# -*- coding: utf-8 -*-
import datetime, time, json, re, os
#from pwd import getpwnam
#quality
str_quality = ''.join(['{StartTime}|||{SubscriberID}', \
'|'*11, '{Bitrate}|{MOSAvg}|{DF}|{MLR}||||', '{CPUUsageHistogram}|', \
'{ProfilePlayoutSecondsHistogram_M}', '|'*8, '{ProfilePlayoutSecondsHistogram_m}', \
'|'*8, '{PlayDuration_M}|{BadDuration_M}|{PlayDuration_S}|{BadDuration_S}||||', \
'{CurrentPlaySuccNum_CurrentPlayErrNum}|{CurrentPlaySuccNum}', '|'*11, \
'{TVGuideReqNum}|{TVGuideSuccessNum}|{TVGuideDelayAvg}|||||', \
'{StallingCount_S}|{StallingDuration_S}||', \
'{BufferTrack_A}|{BufferTrack_M}|{BufferTrack_m}|||', \
'{RTTTrack_A}|{RTTTrack_M}|{RTTTrack_m}|{LostRatioTrack_A}|{LostRatioTrack_M}|{LostRatioTrack_m}', \
'|'*7, '{RAMUsageHistogram_A}|{RAMUsageHistogram_M}|{RAMUsageHistogram_m}|{DiskUsage}', \
'|'*6, '{DownloadSpeed_A}|{DownloadSpeed_M}|{DownloadSpeed_m}||||', \
'{UpBandWidthHistogram_A}|{UpBandWidthHistogram_M}|{UpBandWidthHistogram_m}|', \
'{DownBandWidthHistogram_A}|{DownBandWidthHistogram_M}|{DownBandWidthHistogram_m}||', \
'{URL}|{ServiceType}|{URL_IP}||||', '{DeviceSupplier}||||\n'])
#behavior
str_behavior = ''.join(['{StartTime}|{SubscriberID}', \
'|'*7, '{ServiceType}', '|'*9, '{PlayDuration_M}', \
'|'*19, '{URL}|{URL_IP}||||\n'])
#CPU, RAM
c_r = [0.1, 0.35, 0.6, 0.75, 0.85, 0.95]
#up
up = [125, 375, 750, 1500, 3000, 5000]
#down, ProfilePlayoutSecondsHistogram
down = [1000, 3500, 7500, 15000, 35000, 60000]
pattern = re.compile(r'http://(\d+.\d+.\d+.\d+)/.*')
#get the log file path
def get_log_path(path):
for fpathe,dirs,fs in os.walk(path):
for f in fs:
if f.endswith('.log'):
return os.path.join(fpathe, f)
def get_his_MIN(lis):
for x in lis:
if x != 0:
return lis.index(x)
return 0
def get_his_MAX(lis):
for x in lis[::-1]:
if x != 0:
return lis.index(x)
return 0
#CPU, RAM, up down
def get_his_AVE_R_C(lis):
s = int(100*(0.1*lis[0]+0.35*lis[1]+0.6*lis[2]+0.75*lis[3]+0.85*lis[4]+0.95*lis[5]))
if sum(lis) != 0:
return int (s/ sum(lis))
else:
return 0
def get_up_AVE(lis):
s = int(125*lis[0]+375*lis[1]+750*lis[2]+1500*lis[3]+3000*lis[4]+5000*lis[5])
if sum(lis) != 0:
return int(s / sum(lis))
else:
return 0
def get_down_AVE(lis):
s = int(1000*lis[0]+3500*lis[1]+7500*lis[2]+15000*lis[3]+35000*lis[4]+60000*lis[5])
if sum(lis) != 0:
return int(s / sum(lis))
else:
return 0 #get max PlayDuration time < 300
def get_max_PlayDuration(lis_b_p):
lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300]
if not lis_b_p_:
return 300
else:
return max(lis_b_p_, key=lambda x:x[0])[1] def get_sum_PlayDuration(lis_p):
lis_p_ = [x for x in lis_p if x <= 300]
if sum(lis_p_) <= 300:
return sum(lis_p_)
return 300
#get the max badduration value，make sure the value is smaller than playduration
def get_max_BadDuration(lis_b_p):
lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300]
if not lis_b_p_:
return 0
else:
return max(lis_b_p_, key=lambda x:x[0])[0] def get_sum_BadDuration(lis_b, sum_p):
lis_b_ = [x for x in lis_b if x <= 300]
if sum(lis_b_) < sum_p:
return sum(lis_b_)
else:
return sum_p #recursively get all the dict of a json line
def get_outcome_iter(dict_):
for key, value in dict_.items():
if isinstance(value, dict):
yield from get_outcome_iter(value)
elif isinstance(value, list):
for v in value:
yield from get_outcome_iter(v)
else:
yield key, value def get_dict_str(json_2_dict):
str_quality_tmp, str_behavior_tmp = str_quality, str_behavior
str_quality_dict = dict(StartTime='', SubscriberID='',
Bitrate='', MOSAvg='', DF='', MLR='', CPUUsageHistogram='',
ProfilePlayoutSecondsHistogram_M='', ProfilePlayoutSecondsHistogram_m='',
PlayDuration_M='', BadDuration_M='', PlayDuration_S='', BadDuration_S='',
CurrentPlaySuccNum_CurrentPlayErrNum='', CurrentPlaySuccNum='',
TVGuideReqNum='', TVGuideSuccessNum='', TVGuideDelayAvg='',
StallingCount_S='', StallingDuration_S='',
BufferTrack_A='', BufferTrack_M='', BufferTrack_m='',
RTTTrack_A='', RTTTrack_M='', RTTTrack_m='',
LostRatioTrack_A='', LostRatioTrack_M='', LostRatioTrack_m='',
RAMUsageHistogram_A='', RAMUsageHistogram_M='', RAMUsageHistogram_m='', DiskUsage='',
DownloadSpeed_A='', DownloadSpeed_M='', DownloadSpeed_m='',
UpBandWidthHistogram_A='', UpBandWidthHistogram_M='', UpBandWidthHistogram_m='',
DownBandWidthHistogram_A='', DownBandWidthHistogram_M='', DownBandWidthHistogram_m='',
URL='', ServiceType='', URL_IP='', DeviceSupplier='') str_behavior_dict = dict(StartTime='', SubscriberID='',ServiceType='',
PlayDuration_M='', URL='', URL_IP='') str_dict_tmp = dict(StartTime=[], SubscriberID=[], Bitrate=[],
MOSAvg=[], DF=[], MLR=[], CPUUsageHistogram=[],
ProfilePlayoutSecondsHistogram=[], PlayDuration=[], BadDuration=[],
CurrentPlaySuccNum=[], CurrentPlayErrNum=[],
TVGuideReqNum=[], TVGuideSuccessNum=[], TVGuideDelayAvg=[],
StallingCount=[], StallingDuration=[],
BufferTrack=[], RTTTrack=[], LostRatioTrack=[], RAMUsageHistogram=[],
DiskUsage=[], DownloadSpeed=[], UpBandWidthHistogram=[], DownBandWidthHistogram=[],
URL=[], ServiceType=[], DeviceSupplier=[]) Histogram_list = ['CPUUsageHistogram', 'RAMUsageHistogram', 'UpBandWidthHistogram',
'ProfilePlayoutSecondsHistogram', 'DownBandWidthHistogram',
'RTTTrack', 'LostRatioTrack', 'BufferTrack'] line_dict = get_outcome_iter(json_2_dict)
for key, value in line_dict:
if value == None or value == '':
continue
if key in Histogram_list:
value = [int(x) for x in value.split(',')]
if key in str_dict_tmp:
str_dict_tmp[key].append(value) if str_dict_tmp['StartTime']:
str_quality_dict['StartTime'] = str_dict_tmp['StartTime'][0]
str_behavior_dict['StartTime'] = str_dict_tmp['StartTime'][0]
if str_dict_tmp['SubscriberID']:
str_quality_dict['SubscriberID'] = str_dict_tmp['SubscriberID'][0]
str_behavior_dict['SubscriberID'] = str_dict_tmp['SubscriberID'][0]
if str_dict_tmp['Bitrate']:
str_quality_dict['Bitrate'] = round(max(str_dict_tmp['Bitrate']) / 1024, 2)
if str_dict_tmp['MOSAvg']:
str_quality_dict['MOSAvg'] = round(str_dict_tmp['MOSAvg'][0] / 10, 1)
if str_dict_tmp['DF']:
str_quality_dict['DF'] = int(str_dict_tmp['DF'][0] / 1000)
if str_dict_tmp['MLR']:
str_quality_dict['MLR'] = round(str_dict_tmp['MLR'][0] / 1000000, 1)
if str_dict_tmp['CPUUsageHistogram']:
str_quality_dict['CPUUsageHistogram'] = get_his_AVE_R_C(str_dict_tmp['CPUUsageHistogram'][0])
if str_dict_tmp['ProfilePlayoutSecondsHistogram']:
str_quality_dict['ProfilePlayoutSecondsHistogram_M'] = \
down[get_his_MAX(str_dict_tmp['ProfilePlayoutSecondsHistogram'][0])]
str_quality_dict['ProfilePlayoutSecondsHistogram_m'] = \
down[get_his_MIN(str_dict_tmp['ProfilePlayoutSecondsHistogram'][0])] if str_dict_tmp['PlayDuration'] and str_dict_tmp['BadDuration']:
lis_b_p = list(zip(str_dict_tmp['BadDuration'], str_dict_tmp['PlayDuration']))
str_quality_dict['PlayDuration_M'] = get_max_PlayDuration(lis_b_p)
str_quality_dict['PlayDuration_S'] = get_sum_PlayDuration(str_dict_tmp['PlayDuration'])
str_behavior_dict['PlayDuration_M'] = str_quality_dict['PlayDuration_M']
str_quality_dict['BadDuration_M'] = get_max_BadDuration(lis_b_p)
str_quality_dict['BadDuration_S'] = \
get_sum_BadDuration(str_dict_tmp['BadDuration'], str_quality_dict['PlayDuration_S']) if str_dict_tmp['CurrentPlaySuccNum'] and str_dict_tmp['CurrentPlayErrNum']:
str_quality_dict['CurrentPlaySuccNum_CurrentPlayErrNum'] = \
str_dict_tmp['CurrentPlaySuccNum'][0] + str_dict_tmp['CurrentPlayErrNum'][0]
str_quality_dict['CurrentPlaySuccNum'] = str_dict_tmp['CurrentPlaySuccNum'][0]
if str_dict_tmp['TVGuideReqNum']:
str_quality_dict['TVGuideReqNum'] = str_dict_tmp['TVGuideReqNum'][0]
if str_dict_tmp['TVGuideSuccessNum']:
str_quality_dict['TVGuideSuccessNum'] = str_dict_tmp['TVGuideSuccessNum'][0]
if str_dict_tmp['TVGuideDelayAvg']:
str_quality_dict['TVGuideDelayAvg'] = str_dict_tmp['TVGuideDelayAvg'][0]
if str_dict_tmp['StallingDuration']:
str_quality_dict['StallingDuration_S'] = sum(str_dict_tmp['StallingDuration'])
if str_dict_tmp['StallingCount']:
str_quality_dict['StallingCount_S'] = sum(str_dict_tmp['StallingCount'])
if str_dict_tmp['BufferTrack']:
str_quality_dict['BufferTrack_A'] = sum(str_dict_tmp['BufferTrack'][0])
str_quality_dict['BufferTrack_M'] = max(str_dict_tmp['BufferTrack'][0])
str_quality_dict['BufferTrack_m'] = min(str_dict_tmp['BufferTrack'][0])
if str_dict_tmp['RTTTrack']:
str_quality_dict['RTTTrack_A'] = sum(str_dict_tmp['RTTTrack'][0])
str_quality_dict['RTTTrack_M'] = max(str_dict_tmp['RTTTrack'][0])
str_quality_dict['RTTTrack_m'] = min(str_dict_tmp['RTTTrack'][0])
if str_dict_tmp['LostRatioTrack']:
str_quality_dict['LostRatioTrack_A'] = \
sum(str_dict_tmp['LostRatioTrack'][0]) / (1000000 * len(str_dict_tmp['LostRatioTrack'][0]))
str_quality_dict['LostRatioTrack_M'] = max(str_dict_tmp['LostRatioTrack'][0]) / 1000000
str_quality_dict['LostRatioTrack_m'] = min(str_dict_tmp['LostRatioTrack'][0]) / 1000000
if str_dict_tmp['RAMUsageHistogram']:
str_quality_dict['RAMUsageHistogram_A'] = get_his_AVE_R_C(str_dict_tmp['RAMUsageHistogram'][0])
str_quality_dict['RAMUsageHistogram_M'] = int(c_r[get_his_MAX(str_dict_tmp['RAMUsageHistogram'][0])] * 100)
str_quality_dict['RAMUsageHistogram_m'] = int(c_r[get_his_MIN(str_dict_tmp['RAMUsageHistogram'][0])] * 100)
if str_dict_tmp['DiskUsage']:
str_quality_dict['DiskUsage'] = str_dict_tmp['DiskUsage'][0]
if str_dict_tmp['DownloadSpeed']:
str_quality_dict['DownloadSpeed_A'] = sum(str_dict_tmp['DownloadSpeed']) / len(str_dict_tmp['DownloadSpeed'])
str_quality_dict['DownloadSpeed_M'] = max(str_dict_tmp['DownloadSpeed'])
str_quality_dict['DownloadSpeed_m'] = min(str_dict_tmp['DownloadSpeed'])
if str_dict_tmp['UpBandWidthHistogram']:
str_quality_dict['UpBandWidthHistogram_A'] = get_up_AVE(str_dict_tmp['UpBandWidthHistogram'][0])
str_quality_dict['UpBandWidthHistogram_M'] = up[get_his_MAX(str_dict_tmp['UpBandWidthHistogram'][0])]
str_quality_dict['UpBandWidthHistogram_m'] = up[get_his_MIN(str_dict_tmp['UpBandWidthHistogram'][0])]
if str_dict_tmp['DownBandWidthHistogram']:
str_quality_dict['DownBandWidthHistogram_A'] = get_down_AVE(str_dict_tmp['DownBandWidthHistogram'][0])
str_quality_dict['DownBandWidthHistogram_M'] = down[get_his_MAX(str_dict_tmp['DownBandWidthHistogram'][0])]
str_quality_dict['DownBandWidthHistogram_m'] = down[get_his_MIN(str_dict_tmp['DownBandWidthHistogram'][0])]
if str_dict_tmp['URL']:
URL_index = str_dict_tmp['PlayDuration'].index(str_quality_dict['PlayDuration_M']) \
if str_quality_dict['PlayDuration_M'] != 300 else 0
str_quality_dict['URL'] = str_dict_tmp['URL'][URL_index]
str_behavior_dict['URL'] = str_quality_dict['URL']
str_quality_dict['URL_IP'] = pattern.search(str_quality_dict['URL']).group(1)
str_behavior_dict['URL_IP'] = str_quality_dict['URL_IP']
if str_dict_tmp['ServiceType']:
if str_dict_tmp['ServiceType'][0] == 'LiveTV':
str_dict_tmp['ServiceType'][0] = 'BTV'
if str_dict_tmp['ServiceType'][0] != 'BTV' and str_dict_tmp['ServiceType'][0] != 'VOD':
str_dict_tmp['ServiceType'][0] = 'PLAYBACK'
str_quality_dict['ServiceType'] = str_dict_tmp['ServiceType'][0]
str_behavior_dict['ServiceType'] = str_quality_dict['ServiceType']
if str_dict_tmp['DeviceSupplier']:
str_quality_dict['DeviceSupplier'] = str_dict_tmp['DeviceSupplier'][0] str_quality_tmp = str_quality_tmp.format(**str_quality_dict)
str_behavior_tmp = str_behavior_tmp.format(**str_behavior_dict)
return str_quality_tmp, str_behavior_tmp def get_outcome_files(path_file_source, path_outcome):
if path_file_source != None:
ftime = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
path_file_outcome = path_outcome + 'isa_stb_quality_' + ftime + '.dat'
with open(path_file_outcome, 'a') as f_quality, \
open(path_file_source, 'r') as f:
for line in f:
if line == '<==SPLIT==>\n':
continue
json_2_dict = json.loads(line)
outcome = get_dict_str(json_2_dict)
f_quality.write(outcome[0])
#appserver_uid = getpwnam('appserver')[2]
#appserver_gid = getpwnam('appserver')[3]
#os.chown(path_file_outcome, appserver_uid, appserver_gid)
#os.remove(path_file_source)#delete log files if __name__ == '__main__':
#path_source = '/home/omc/'#folder contains log file
#path_outcome = '/srv/smartcare/share/data/fbb/src/vcem/xDR'#folder contains dat file
path_source = 'C:\\Users\\l00429182\\Desktop\\'
path_outcome = 'C:\\Users\\l00429182\\Desktop\\'
path_file_source = get_log_path(path_source)#log files that found in 5 mins
print(path_file_source)
get_outcome_files(path_file_source, path_outcome)