使用ganglia监控mongodb数据库collection的增长

    同事有个需求，需要监控mongodb数据库中某一个collection的数据增长情况，原因是由于有定时任务不断往数据库里填充数据，如果定时任务挂掉了可以及时重启任务，保证数据的准确性。其实也可以监控定时任务的执行情况，之后再研究。
    跟博客：使用ganglia监控mongodb集群   文章介绍的一样，使用python脚本来扩展，所以也就不多说，直接上传脚本代码
1. 创建mongodb_collection.pyconf脚本（collection根据情况换成表名）

点击(此处)折叠或打开

1. # mongodb_collection.pyconf
2. modules {
3.     module {
4.         name = "mongodb_EPGInfo_www"    #EPGInfo表示数据库名，www表示collection的名字
   
5.         language = "python"
6.         param total_count{
7.             value = "/opt/mongodb/bin/mongo --host 10.9.201.190 --port 27017 EPGInfo --quiet --eval 'printjson(db.www.find().count())'"            #语句的作用是获得www表中的记录数
   
8.         }
9.     }
10. }
11. collection_group {
12.     collect_every = 30
13.     time_threshold = 90
14.     metric {
15.         name = "www_count_total"
16.         title = "Total Count"
17.     }
18.     metric {
19.         name = "www_count_insert"
20.         title = "Insert per Sec"
21.     }
22. }

    具体说明看代码中的注释
2. 创建mongodb_collection.py脚本

点击(此处)折叠或打开

1. #!/usr/bin/env python
3. import os
4. import json
5. import string
6. import time
7. import copy
8. import re
10. PARAMS = {
11.     "total_count" : "/opt/mongodb/bin/mongo --host 10.9.201.190 --port 27017 EPGInfo --quiet --eval 'printjson(db.www.find().count())'"
12. }
14. NAME_PREFIX = 'www_'
16. METRICS = {
17.     'time' : 0,
18.     'data' : {}
19. }
21. LAST_METRICS = copy.deepcopy(METRICS)
22. METRICS_CACHE_TTL = 3
24. def get_metrics():
25.     """Return all metrics"""
26.     global METRICS, LAST_METRICS
27.     if (time.time() - METRICS['time']) > METRICS_CACHE_TTL:
28.         metrics = {}
29.     # get raw metric data
30.     o = os.popen(PARAMS['total_count'])
31.     # clean up
32.     metrics_str = ''.join(o.readlines()).strip() # convert to string
33.     metrics_str = re.sub('\w+\((.*)\)', r"\1", metrics_str) # remove functions
35.     metrics['count_total']=float(json.loads(metrics_str))     #增加监控字典
    
36.     metrics['count_insert']=metrics['count_total']            #增加监控字典
    
37.     # update cache
38.     LAST_METRICS = copy.deepcopy(METRICS)
39.     METRICS = {
40.         'time': time.time(),
41.         'data': metrics
42.     }
43.     print(METRICS)
44.     return [METRICS, LAST_METRICS]
46. def get_value(name):
47.     """Return a value for the requested metric"""
48.     # get metrics
49.     metrics = get_metrics()[0]
50.     print(metrics)
51.     # get value
52.     name = name[len(NAME_PREFIX):] # remove prefix from name
53.     try:
54.         result = metrics['data'][name]
55.     except StandardError:
56.         result = 0
57.     return result
59. def get_rate(name):
60.     """Return change over time for the requested metric"""
61.     # get metrics
62.     [curr_metrics, last_metrics] = get_metrics()
63.     # get rate
64.     name = name[len(NAME_PREFIX):] # remove prefix from name
65.     try:
66.         rate = float(curr_metrics['data'][name] - last_metrics['data'][name]) / \
67.         float(curr_metrics['time'] - last_metrics['time'])
68.         if rate < 0:
69.             rate = float(0)
70.     except StandardError:
71.         rate = float(0)
72.     return rate
74. def get_total_count(name):
75.     total_count = get_value(name)
76.     return total_count
78. def get_count_insert(name):
79.     count_insert = get_rate(name)
80.     return count_insert
82. def metric_init(lparams):
83.     global PARAMS
85.     # set parameters
86.     for key in lparams:
87.         PARAMS[key] = lparams[key]
89.     # define descriptors
90.     time_max = 60
91.     groups = 'EPGInfo_www'
92.     descriptors = [
93.         {
94.             'name': NAME_PREFIX + 'count_total',
95.             'call_back': get_total_count,
96.             'time_max': time_max,
97.             'value_type': 'float',
98.             'units': 'Total',
99.             'slope': 'both',
100.             'format': '%f',
101.             'description': 'Total Count',
102.             'groups': groups
103.         },
104.         {
105.             'name': NAME_PREFIX + 'count_insert',
106.             'call_back': get_count_insert,
107.             'time_max': time_max,
108.             'value_type': 'float',
109.             'units': 'Inserts/Sec',
110.             'slope': 'both',
111.             'format': '%f',
112.             'description': 'Insert per Sec',
113.             'groups': groups
114.         }
115.     ]
116.     return descriptors
118. def metric_cleanup():
119.     pass
121. if __name__ == '__main__':
122.     descriptors = metric_init(PARAMS)
123.     while True:
124.         for d in descriptors:
125.             print (('%s = %s') % (d['name'], d['format'])) % (d['call_back'](d['name']))
126.         print ''
127.         time.sleep(METRICS_CACHE_TTL)

    具体说明看代码中的注释