pycurl之公共方法--请求/上传/下载，解析json

  1 # -*- coding=utf-8 -*-
  2 #curl公共程序
  3 #张明伟 20200331
  4 import pycurl
  5 from io import BytesIO
  6 import  datetime,time,os
  7 import  logging
  8 import json
  9 from logging import handlers
 10 testdebug=1 #测试模式：1为是，0为否
 11 if testdebug:
 12     ip = ‘221.122.125.207‘ #测试环境IP
 13 else:
 14     ip = ‘221.122.125.201‘ #正式环境IP
 15 
 16 pth=os.getcwd()+os.sep+‘data‘+os.sep
 17 logpath =os.getcwd() + os.sep + ‘logs‘ + os.sep
 18 
 19 #返回当前日期
 20 def dtnow():
 21     return datetime.datetime.today().strftime(‘%Y%m%d‘)
 22 
 23 #返回文件名相关信息
 24 def filestr(filename):
 25     filepath,tmpfile=os.path.split(filename)
 26     basename,extention=os.path.splitext(tmpfile)
 27     return filepath,tmpfile,basename,extention
 28 
 29 #暂时不用
 30 def init():
 31     dt=(datetime.datetime.today()+datetime.timedelta(days=-1)).strftime(‘%Y-%m-%d‘)
 32     rdt=dt.replace(‘-‘,‘‘)
 33     logfile=‘bigdata‘+rdt+‘.log‘
 34     logging.basicConfig(level=logging.INFO,format=‘%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s‘,filename=logfile)
 35     logging.info("系统金额我发你")
 36     print(os.path.exists(logfile))
 37 
 38 #curlcmd用于上传下载，scp文件，http请求
 39 #op=download/upload 下载或上传 默认是下载，上传时需要传入filename,http请求时需要data
 40 def curlcmd(url,op="download",filename=None,data=""):
 41     if not filename:
 42         filename=‘‘
 43     c = pycurl.Curl()
 44     c.setopt(pycurl.SSL_VERIFYPEER, False)  #ssl不信任或无证书时需要设置这个 等同于curl -k/--insecure 参数
 45     c.setopt(pycurl.SSL_VERIFYHOST, False)  #
 46     c.setopt(pycurl.URL, url+filename)  #上传时需要指定上传文件名，所以filename加入到url中
 47     c.fp = BytesIO()    #设置输出
 48     c.setopt(pycurl.POST, 1)  #以POST方式提交
 49     # c.setopt(pycurl.POSTFIELDS, urllib.urlencode(data))
 50     if isinstance(data,list): #提交大量数据
 51         c.setopt(pycurl.HTTPPOST, data)
 52     elif isinstance(data,dict): #json格式的数据上传 对应着大量的表单数据提交
 53         c.setopt(pycurl.POSTFIELDS,json.dumps(data))
 54     c.setopt(pycurl.USERPWD, "ftpuser:Cbpp@123") #用户名密码
 55     c.setopt(pycurl.WRITEFUNCTION, c.fp.write) #curl执行结果
 56     
 57     #print(data,url,op,filename)
 58     if op=="upload": #上传文件
 59         if not filename:
 60             raise Exception("请输入上传文件名!")
 61         filename1=pth+filename
 62         filename1size=os.path.getsize(filename1)
 63         c.setopt(pycurl.UPLOAD, True) #设置上传标志
 64         c.setopt(pycurl.INFILESIZE, filename1size) #上传需要传入文件大小标志
 65         # c.setopt(pycurl.READFUNCTION, open(filename1,"rd").read(filename1size))
 66         c.setopt(pycurl.READFUNCTION, open(filename1, "rb").read)
 67     # print(data, ‘@@@@@@‘)
 68     # print(urllib.urlencode(data))
 69     c.perform()  #curl中真正的执行
 70     code = c.getinfo(c.HTTP_CODE) #获取返回码
 71     html = c.fp.getvalue()   #获取返回结果
 72     print(html)
 73     return {‘code‘:code,‘html‘:html} #返回json格式
 74 
 75 #日志类
 76 class Logger(object):
 77     level_relations={
 78         ‘debug‘:logging.DEBUG,
 79         ‘info‘:logging.INFO,
 80         ‘waining‘:logging.WARNING,
 81         ‘error‘:logging.ERROR,
 82         ‘crit‘:logging.CRITICAL
 83     }
 84 
 85     def __init__(self,filename=None,level=‘debug‘,when=‘D‘,backCount=3,fmt=‘%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s‘):
 86         if not os.path.exists(logpath):
 87             os.mkdir(logpath)
 88         if filename is None:
 89             filename=filestr(__file__)[2]  #返回无后缀的文件名
 90         logfile=logpath+filename+dtnow()+‘.log‘
 91         self.logger=logging.getLogger(logfile)
 92         format_str=logging.Formatter(fmt)
 93         self.logger.setLevel(self.level_relations.get(level))
 94         sh=logging.StreamHandler()
 95         sh.setFormatter(format_str)
 96         th=handlers.TimedRotatingFileHandler(filename=logfile,when=when,backupCount=backCount)
 97         th.setFormatter(format_str)
 98         self.logger.addHandler(sh)
 99         self.logger.addHandler(th)
100 
101 #json数据类型解析
102 class expressDict():
103     fldlist = []  # type:list  #字段名列表
104     valuelist = []  # type:list #值列表
105     sql = ""  #SQL INSERT语句列表
106     valuesql = "" #SQL值语句列表
107 
108     def __init__(self,jsstr,sqlquote="‘"):
109         self.jsstr = jsstr
110         self.sqlquote = "‘"
111         self.tabname = ""
112         self.sqls=[]
113         self.sqlFlag = True  #默认是需要得到SQL语句
114         # self.fldlist=[] #type:list
115         # self.valuelist=[] #type:list
116         # self.sql = ""
117         # self.valuesql = ""
118 
119     #数据处理后再次重置数据
120     def __clear(self):
121         self.fldlist = []  # type:list
122         self.valuelist = []  # type:list
123         self.sql = ""
124         self.valuesql = ""
125     
126     def __display_item(self, jsstr):
127         """
128         根据json格式自动生成SQL语句或文件:递归
129         文件存放在{pth}目录下
130         sqlFlag默认为真，为真时产生sql语句，否则生成脚本
131         :param jsstr:
132         :return:
133         """
134         #文件格式存放时不需要分割符
135         if not self.sqlFlag:
136             self.sqlquote=""
137         #如果json格式中数据库字典格式，则把键值做为表名
138         if isinstance(jsstr, dict):
139             for k,v in jsstr.items():
140                 if not isinstance(v,list):
141                     self.fldlist.append(k)
142                     self.valuelist.append(v)
143                     # print(k,‘--->‘,v)
144                     self.sql=self.sql+k+‘,‘
145                     self.valuesql=self.valuesql+self.sqlquote+v+self.sqlquote+‘,‘
146                 else:
147                     if not k:
148                         # print(self.tabname,"*" * 20)
149                         pass
150                     else:
151                         if k == self.tabname:
152                             self.sql = self.sql+‘) ‘
153                         self.tabname=k
154                 # print(self.tabname+‘@@@@@@@‘)
155                 self.__display_item(v)
156         # 如果json格式中数据库列表格式，则把其他数据做库数据值
157         elif isinstance(jsstr, list):
158             for jslst in jsstr:
159                 if self.sqlFlag:
160                     self.sql=self.sql+"insert into %s (" %(self.tabname)
161                     self.__display_item(jslst)
162                     self.sql = self.sql.strip(‘,‘) + ") values ("+self.valuesql.strip(‘,‘)+");"
163                     self.sqls.append(self.sql)
164                     self.__clear()
165                 else:
166                     finame = pth + self.tabname + ‘_‘ + dtnow() + ‘.del‘
167                     self.__display_item(jslst)
168                     data = self.valuesql.strip(‘,‘) + os.linesep  # type:str
169                     self.__clear()
170                     print(finame, data)
171                     with open(finame, ‘a+‘) as fp:
172                         fp.write(data.encode("utf-8"))
173 
174     def display_items(self,sqlFlag=True):
175         # return self.display_item(self.jsstr)
176         self.sqlFlag=sqlFlag
177         return self.__display_item(self.jsstr)
178 
179     def dispay_sqls(self):
180         for sql in self.sqls:
181             print(sql)
182         # print(self.valuesql)
183 
184 # curl -k -v -T "{table_name1_20200403.del,table_name2_20200403.del}" -u ftpuser scp://21.12.2.196/tmp/
185 # print(scptools(url="scp://21.12.2.196//tmp/gjj20200402.log"))
186 # print(scptools(url="scp://ftpdata@21.12.2.196//tmp/"))
187 # print(curlcmd(data="",url="scp://21.12.2.196//tmp/gjj20200402.log"))
188 # print(curlcmd(data="",url="scp://ftpdata@21.12.2.196/tmp/",op="upload",filename="test111.txt"))