1、日志下载代码(cdn.py)(请在Linux系统下运行)
#!/usr/bin/python2.7
# -*- coding:utf-8 -*- import sys,os,gzip,json,requests,urllib
import base64,hmac,time,uuid,ConfigParser
from hashlib import sha1 download_path=os.path.join(os.path.dirname(os.path.abspath(__file__)),"download")
cdn_server_address = 'https://cdn.aliyuncs.com' class read_config(object):
'''
解析配置文件
'''
def __init__(self):
self._CONFIGFILE=os.path.join(os.path.dirname(os.path.abspath(__file__)), "aliyun.ini")
self._config=ConfigParser.ConfigParser()
self._config.read(self._CONFIGFILE)
self._access_id = self._config.get('Credentials', 'accesskeyid')
self._access_key = self._config.get('Credentials', 'accesskeysecret')
self._Action = self._config.get('Must', 'Action')
self._DomainName = self._config.get('Must', 'DomainName')
self._Must_list=self._config.items('Select')
self._user_param={} @property
def access_key_id(self):
return self._access_id @property
def access_key_secret(self):
return self._access_key @property
def user_params(self):
if self._Action and self._DomainName:
self._user_param['Action'] = self._Action
self._user_param['DomainName'] = self._DomainName
for i in self._Must_list:
self._user_param[i[0]] = i[1]
return self._user_param class read_write(object):
'''
保存已下载过的日志
'''
def __init__(self):
self._logfilename = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".download") @property
def read(self):
try:
with open(self._logfilename, 'rb', ) as f:
logfile = f.read()
logfile = json.loads(logfile)
if len(logfile) > 20:
logfile.pop(0)
return logfile
except IOError as f:
return [] def write(self,logfile):
with open(self._logfilename, "wb") as f:
file = json.dumps(logfile)
f.write(file) class GZipTool(object):
"""
压缩与解压gzip
""" def __init__(self, bufSize=1024 * 8):
self.bufSize = bufSize
self.fin = None
self.fout = None def compress(self, src, dst):
self.fin = open(src, 'rb')
self.fout = gzip.open(dst, 'wb')
self.__in2out() def decompress(self, gzFile, dst):
self.fin = gzip.open(gzFile, 'rb')
self.fout = open(dst, 'wb')
self.__in2out() def __in2out(self, ):
while True:
buf = self.fin.read(self.bufSize)
if len(buf) < 1:
break
self.fout.write(buf)
self.fin.close()
self.fout.close() def percent_encode(str):
res = urllib.quote(str.decode('UTF-8').encode('utf8'), '')
res = res.replace('+', '%20')
res = res.replace('*', '%2A')
res = res.replace('%7E', '~')
return res def compute_signature(parameters, access_key_secret):
'''
:param parameters: 生成签名需要的数据
:param access_key_secret: 访问阿里云需要的key
:return: 返回签名信息
'''
sortedParameters = sorted(parameters.items(), key=lambda parameters: parameters[0])
canonicalizedQueryString = ''
for (k,v) in sortedParameters:
canonicalizedQueryString += '&' + percent_encode(k) + '=' + percent_encode(v)
stringToSign = 'GET&%2F&' + percent_encode(canonicalizedQueryString[1:])
h = hmac.new(access_key_secret + "&", stringToSign, sha1)
signature = base64.encodestring(h.digest()).strip()
return signature def compose_url(readconfig):
'''
:param user_params: 生成第一次请求URL所需要的参数
:param readconfig: 配置文件对象
:return: 第一次请求的URL
'''
timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
parameters = { \
'Format' : 'JSON', \
'Version' : '2014-11-11', \
'AccessKeyId' : readconfig.access_key_id, \
'SignatureVersion' : '1.0', \
'SignatureMethod' : 'HMAC-SHA1', \
'SignatureNonce' : str(uuid.uuid1()), \
'TimeStamp' : timestamp, \
}
for key in readconfig.user_params.keys():
if readconfig.user_params[key]:
parameters[key] = readconfig.user_params[key]
signature = compute_signature(parameters, readconfig.access_key_secret)
parameters['Signature'] = signature
url = cdn_server_address + "/?" + urllib.urlencode(parameters)
return url def make_request(readconfig,readwrite):
'''
:param user_params: 生成第一次请求URL所需要的参数
:param readconfig: 配置文件对象
:param readwrite: 读取和保存已下载过的文件对象
:return: 返回一个字典，key为日志文件名，value为日志的URL地址
'''
url = compose_url(readconfig)
path_dic={}
try:
res=requests.get(url)
res=res.json()
res=res['DomainLogModel']['DomainLogDetails']['DomainLogDetail']
logfile = readwrite.read
for i in res:
if i['LogName'] not in logfile:
logfile.append(i['LogName'])
path_dic[i['LogName']]=i['LogPath']
readwrite.write(logfile)
return path_dic
except Exception:
return False def download(download_path,readconfig,readwrite):
'''
:param download_path: 日志URL
:param user_params: 生成第一次请求URL所需要的参数
:param readconfig: 配置文件对象
:param readwrite: 读取和保存已下载过的文件对象
:return:
'''
link_path=make_request(readconfig,readwrite)
if link_path:
for name,path in link_path.items():
filename=os.path.join(download_path,name)
logname=filename[:-3]+'.txt'
pathurl="https://%s" %path
r = requests.get(pathurl)
with open(filename, 'wb') as f:
f.write(r.content)
GZipTool().decompress(filename, logname)
os.remove(filename)
os.system("find %s -type f -mtime +7 |xargs rm -rf" % download_path) if __name__ == '__main__':
readconfig = read_config()
readwrite=read_write()
if not readconfig.user_params:
sys.exit(1)
else:
download(download_path,readconfig,readwrite)
2、配置文件内容(aliyun.ini)
[Credentials]
accesskeyid =
accesskeysecret = [Must]
Action =
DomainName = [Select]
LogDay=
PageSize=
PageNumber=
3、文件结构图
4、阿里云日志下载API接口
https://help.aliyun.com/document_detail/27224.html?spm=a2c4g.11186623.6.704.vz36cV
https://docs-aliyun.cn-hangzhou.oss.aliyun-inc.com/cn/cdn/0.1.99/assets/api/callmethod_sdk_python.zip #python版本签名生成代码