Date: 20140207
Auth: Jin
设置一个LOG收集系统
1. 收集原生（不解析，不压缩）的业务日志和WEB日志（NGINX,PHP）
2. 提供给开发，测试直接阅读和下载
需求分析
原生日志，所以不需要其他程序介入，需要收集和阅读下载，具体分析开发测试人员拿到日子自行处理
1、收集可以通过FTP采集
2、阅读和下载可以通过WEB形式
3、收集周期可以通过crontab控制脚本实现, 初步设置为每小时
4、核心是收集日子的脚本，考虑到扩展和重复使用，我使用python编写d
步骤
一、搭建FTP服务器
1.install
yum install pure-ftpd
2.config
mkdir /data/ftproot/logs
chown webroot.webroot /data/ftproot/logs
vim /etc/pure-ftpd/pure-ftpd.conf
注意流量参数
Bind 10.0.0.221,21
ChrootEveryone yes
BrokenClientsCompatibility yes
Daemonize yes
MaxClientsPerIP 20
VerboseLog yes
DisplayDotFiles no
AnonymousOnly no
NoAnonymous yes
SyslogFacility none
DontResolve yes
MaxIdleTime 15
LimitRecursion 10000 8
AnonymousCanCreateDirs no
MaxLoad 4
PassivePortRange 45000 50000
#AnonymousRatio 1 10
#UserRatio 1 10
AntiWarez yes
#AnonymousBandwidth 200
UserBandwidth 8
Umask 133:02
MinUID 100
AllowUserFXP no
AllowAnonymousFXP no
ProhibitDotFilesWrite no
ProhibitDotFilesRead no
AutoRename yes
AnonymousCantUpload yes
AltLog clf:/var/log/pureftpd.log
PureDB /etc/pure-ftpd/pureftpd.pdb
MaxDiskUsage 99
CreateHomeDir no
CustomerProof yes
3.start ftp seriver
/etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]
chkconfig pure-ftpd on
4.add account
# pure-pw useradd logsftp -u 9999 -g 9999 -d /data/ftproot/logs
Password:
Enter it again:
# pure-pw mkdb
# /etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]
5.test
on opensuse desktop test
# zypper install lftp
lftp 10.0.0.221
lftp 10.0.0.221:~> user logsftp
Password:
lftp [email protected]:~> ls
ls: Login failed: 530 ��֤ʧ�ܣ���Ǹ
lftp [email protected]:~>
二、搭建WEB服务器
DOCMENTROOT为
server {
listen 80;
server_name logs.test.com;
root /data/ftproot/logs/;
location / {
index index.htm index.html;
autoindex on;
autoindex_localtime on;
auth_basic "logs access";
auth_basic_user_file log-auth.conf;
error_log off;
access_log /var/log/nginx/log.test.com-access.log;
#access_log off;
}
}注意点：
autoindex_localtime on;
参考：http://blog.chinaunix.net/uid-26719405-id-3508444.html
autoindex_localtime on;
默认为off，显示的文件时间为GMT时间。
改为on后，显示的文件时间为文件的服务器时间
三、代码编写
无LOG版本
加入计划任务
5 * * * * /home/gbin/logcollect.py >> /home/gbin/logcollect.log 2>&1
计划任务报错 登录用户问题
删除userName = os.getlogin()
- 配置文件logcollect.ini
[global]
ip = 127.0.0.1
username = logsftp
password = pwd
mode = 0 [applogs]
dms-log = /home/dms/logs/service.log
store-logdir = /home/store/logs/
stock-logdir = /home/stock/logs/ [weblogs]
nginxlog = /var/log/nginx.log
phplog = /var/log/php-fpm/php-fpm.log
- 函数和对象：gbopt.py
#!/usr/bin/env python
# coding=utf8
'''
Created on 2014-03-05 @author: Jin ''' __version__ = "0.0.1#date:2014-03-05" __all__ = ['readConfig',
'ftpPutGb',
'OptFile'
] import os
import sys
import ConfigParser
import ftplib
import time
import socket
from ftplib import FTP
from time import strftime as printtime
from time import sleep as wait progName = 'logcollect'
userName = os.getlogin()
hostName = socket.gethostname()
pwdDir = os.getcwd()
workDir = os.path.dirname(sys.argv[0]) configFile = progName+'.ini'
todayDate = time.strftime("%Y%m%d", time.localtime()) if workDir == '' or workDir == '.':
progDir = pwdDir
elif workDir.startswith('/'):
progDir = workDir
elif workDir.startswith('./'):
progDir = pwdDir+workDir.lstrip('.')
else:
progDir = pwdDir+'/'+workDir confPath = progDir+'/'+configFile
logPath = progDir+'/'+progName+'.log' def readConfig(filename='',section=''):
'''read ini config return dict'''
Call = 'Call '+sys._getframe().f_code.co_name
if not os.path.isfile(filename):
print "%s: Error, File %s is not exists,Please check it!" % (Call,filename)
sys.exit(10)
else:
try:
Config = ConfigParser.ConfigParser()
Config.read(filename)
sections = Config.sections()
configDict = {}
for conf in Config.items(section):
ckey = conf[0]
cvalue = conf[1]
configDict[ckey] = cvalue
except ConfigParser.ParsingError,e:
print "%s: Read<%s> Section<%s> Parsing Error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n')))
sys.exit(11)
except Exception, e:
print "%s: Read<%s> Section<%s> Exception error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n')))
sys.exit(12)
else:
return configDict def ftpPutGb(filename,inittime,server=None,username=None,password=None,localdir=None,remotedir='testdir',debuglevel=0,model=1,retrytime=10,timeout=120):
"""Put file to FTP server,use local path
exitcode introduce:
timeout | 99
connect failed | 21
user or passwd error | 22
create dir fails | 23
change dir fails | 24
read file fails | 25
transfer file fails | 26
any ftp transfer fails | 27
server or username or password not exist | 28
call ftpPutGb('/home/jin/code/python/tt.ini','127.0.0.1','logsftp','passwd',remotedir='testdir1/testdir2/testdir3',model=0)"""
Call = 'Call '+sys._getframe().f_code.co_name
if not os.path.isfile(filename):
print "Notice: %s is not exists!" % filename
return False
if not localdir:
localdir=os.path.dirname(filename)
filename=os.path.basename(filename)
uploadfile='STOR '+filename
ftp=FTP()
bufsize=1024
os.chdir(localdir)
ftp.set_debuglevel(debuglevel)
remotedirList=remotedir.split('/')
appName=remotedirList[0]
machineName=remotedirList[1]
dayName=remotedirList[2]
if server and username and password:
print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) Begin." % (username,filename,server,remotedir)
try:
ftp.connect(server)
except socket.error:
waittime=inittime
wait(retrytime)
waittime+=retrytime
if waittime==timeout:
print "%s: Put %s to FTP(%s) timeout(%s),exit!" % (Call,filename,server,timeout)
sys.exit(20)
print "Notice: Retry Put %s to FTP(%s) again after %s s" % (filename,server,retrytime)
ftpPutGb(filename,waittime,server,username,password,localdir,remotedir,debuglevel,model,retrytime,timeout)
except ftplib.error_perm,e:
print "Error: Connect to FTP(%s) failed with<%s>" % (server,e)
sys.exit(21)
else:
try:
ftp.login(username,password)
ftp.set_pasv(model)
except ftplib.error_perm,e:
print "Error: User(%s) Login FTP(%s) failed with<%s>" % (username,server,e)
sys.exit(22)
else:
try:
try:
ftp.mkd(appName)
except ftplib.error_perm,e:
pass
finally:
ftp.cwd(appName)
try:
ftp.mkd(machineName)
except ftplib.error_perm,e:
pass
finally:
ftp.cwd(machineName)
try:
ftp.mkd(dayName)
except ftplib.error_perm,e:
pass
finally:
ftp.cwd(dayName)
try:
file_handler = open(filename,'rb')
except ftplib.error_perm,e:
print "Error: User(%s) on FTP(%s) Open file(%s) failed with %s!" % (username,server,filename,e)
sys.exit(25)
else:
try:
ftp.storbinary(uploadfile,file_handler,bufsize)
except ftplib.error_perm,e:
print "Error: User(%s) on FTP(%s) storbinary file(%s) failed with %s!" % (username,server,filename,e)
sys.exit(26)
except Exception, e:
print "Error: User(%s) Put file(%s) to FTP(%s:%s) failed exit with %s!" % (username,filename,server,remotedir,e)
sys.exit(27)
else:
print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) successful." % (username,filename,server,remotedir)
return True
finally:
ftp.set_debuglevel(0)
ftp.close()
else:
print "Error: server(%s) or username(%s) or password(%s) is not exist!" % (server,username,password)
sys.exit(28) class OptFile(file):
'''log file object'''
def __init__(self,path,remotedir):
self.filePath = path
self.remotedir = remotedir
globalDict = readConfig(confPath,'global')
self.ftpIp = globalDict['ip']
self.ftpUser = globalDict['username']
self.ftpPass = globalDict['password']
self.ftpMode = globalDict['mode']
file.__init__
def put(self):
ftpPutGb(self.filePath,0,self.ftpIp,self.ftpUser,self.ftpPass,remotedir=self.remotedir,model=self.ftpMode) def main():
pass
if __name__ == '__main__':
main()
- 调用脚本：logcollect.py
#!/usr/bin/env python
# coding=utf8
'''
Created on 2014-03-05 @author: Jin '''
import os
import sys
import gbopt def collectLogs(logtype):
applogsDict=gbopt.readConfig(gbopt.confPath,logtype)
for l in applogsDict:
appName=l.split('-')[0]
hostName=gbopt.hostName
dayName=gbopt.todayDate
remoteDir=appName+'/'+hostName+'/'+dayName
if l.endswith('dir'):
locallogPath=applogsDict[l]+gbopt.todayDate+'.log'
else:
locallogPath=applogsDict[l]
logfile=gbopt.OptFile(locallogPath,remoteDir)
logfile.put() def main():
for i in ['applogs','weblogs']:
collectLogs(i) if __name__ == '__main__':
main()