Ruby 杂记 加代码练习

前言，基础略过，基本上跟python等差不多，看基础就可以了。

1. json嵌套list的数据提取

1. result = { "api.xxx.cn": [
2. {
3. "address": "10.0.16.19:9022",
4. "ttl": 0
5. }
6. ]
7. }
8. 
   
9. route = JSON.parse(result)
10. 
    
11. temp_route = {}
12. route.each do |host, ip_metas|
13. temp_route[host] = ip_metas.map { |ip_meta| ip_meta["address"]}
14. end
15. ########
    
16. map, _ = build_map  #放弃第二个返回参数
17. 
    

2. 根据条件检索ElasticSearch，找过message里面proxy error的IP数量，然后打印出结果，后续可以写到zabbix等监控工具里。

1. # Name: RouterLogRetriver 
2. 
   
3. #!/usr/bin/env ruby
4. require 'net/http'
   
5. require 'uri'
   
6. require 'openssl'
   
7. require 'json'
   
8. require 'date'
   
9. require 'yaml'
   
10. 
    
11. 
    
12. class RouterLogRetriver
    
13.   class << self                 # 无需生成一个对象然后再调用方法，见 114行。
    
14.     attr_accessor :host, :user, :password, :time_span_minute    # 和Python不同，Ruby的类变量不允许被类方法直接访问，只能通过accessor存取器这种。
    
15.     def configure(host, user, password, time_span_minute)
    
16.       @host = host
    
17.       @user = user
    
18.       @password = password
    
19.       @time_span_minute = time_span_minute
    
20.     end
    
21. 
    
22.     def is_configured?
    
23.       !(@host.nil? || @user.nil? || @password.nil? || @time_span_minute.nil?)
    
24.     end
    
25. 
    
26.     def construct_uri
    
27.       date = DateTime.now.new_offset(0).strftime('%C%y.%m.%d')
    
28.       return URI.parse("http://#{@host}/logstash-#{date}/_search")
    
29.     end
    
30. 
    
31.     def construct_payload
    
32.        timestamp_e = DateTime.now.strftime('%Q')
    
33.        timestamp_s = (timestamp_e.to_i - 1000*60*@time_span_minute).to_s
    
34.        payload = '        # ES可以通过JSON方式查询，下边是构造一个查询体。
    
35. {
    
36.   "query": {
    
37.     "filtered": {
    
38.       "query": {
    
39.         "bool": {
    
40.           "should": [
    
41.             {
    
42.               "query_string": {
    
43.                 "query": "\"proxy error:\"-\"EOF\""
    
44.               }
    
45.             }
    
46.           ]
    
47.         }
    
48.       },
    
49.       "filter": {
    
50.         "bool": {
    
51.           "must": [
    
52.             {
    
53.               "range": {
    
54.                 "@timestamp": {
    
55.                   "from": ' + timestamp_s + ',
    
56.                  "to": ' + timestamp_e + '
    
57.                 }
    
58.               }
    
59.             }
    
60.           ]
    
61.         }
    
62.       }
    
63.     }
    
64.   },
    
65.  "size": 500,
    
66.   "sort": [
    
67.     {
    
68.       "msg_timestamp": {
    
69.         "order": "desc"
    
70.       }
    
71.     }
    
72.   ]
    
73. }
    
74.        '
    
75.        payload    # 等价于 return payload。
    
76.     end
    
77. 
    
78.     def search_result
    
79.       return nil unless is_configured?
    
80.       uri = construct_uri
    
81.       http = Net::HTTP.new(uri.host, uri.port)
    
82.       req = Net::HTTP::Get.new(uri.path)
    
83.       req.basic_auth @user, @password
    
84.       req["Content-Type"]="application/json"
    
85.       req.body = construct_payload
    
86. 
    
87.       errors = {}
    
88. 
    
89.       begin
    
90.         response = http.request(req)
    
91.         result = response.body
    
92. 
    
93.         logs = JSON.parse(result)
    
94.         logs['hits']['hits'].each do |l|
    
95.           error = l['_source']['@message']
    
96.           next if error.nil?
    
97.           match_set = error.match(/.*proxy error: \w* tcp (\d+\.\d+.\d+.\d+:\d+):.*/)
    
98.           if match_set
    
99.             backend = match_set[1]
    
100.             errors[backend] = 0 unless errors.has_key?(backend)
     
101.             errors[backend] += 1
     
102.           end
     
103.         end
     
104.       rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
     
105.         Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
     
106.         puts "[ERR] CAN NOT GET Logstash logs, #{e.message}"
     
107.       end
     
108. 
     
109.       errors
     
110.     end
     
111.   end
     
112. end
     
113. # log_url = ENV['LOGSEARCH_HOST'] || "es.xxx.com" 可以从环境变量取值
     
114. RouterLogRetriver.configure("es.xxx.com","elk","xxx",20)
     
115. print RouterLogRetriver.search_result

3. 一般来说要写个项目，把class、module放./lib下，执行文件放bin下。然后bin下加载

1. libdir = File.expand_path(File.join(File.dirname(__FILE__), "../lib"))
2. $LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)

4. rspec一例，测试代码

1. # spec_helper.rb
   
2. 
   
3. require 'logger'
   
4. LOGGER=Logger.new('/dev/null')
   
5. require 'router_log_retriver.rb'
   
6. 
   
7. 
   
8. $:.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
   
9. 
   
10. SPEC_ROOT = File.expand_path(File.dirname(__FILE__))
    
11. 
    
12. RSpec.configure do |config|
    
13.   config.color = true
    
14.   config.run_all_when_everything_filtered = true
    
15.   config.filter_run :focus
    
16. end
17. 
    
18. ## routerlogretriver.rb    主测试
    
19. require File.expand_path '../spec_helper.rb', __FILE__
    
    describe RouterLogRetriver do
    
      before :each do
        described_class.configure('logsearch.run.xxx.com', 'fake_log_user', 'fake_log_pass', 5)
      end
    
      it 'load configure and populate variables' do
        expect(described_class.host).to eq('logsearch.run.xxx.com')
        expect(described_class.user).to eq('fake_log_user')
        expect(described_class.password).to eq('fake_log_pass')
        expect(described_class.time_span_minute).to eq(5)
      end
    
      it 'construct the logstash uri base on time stamp' do
        allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
        uri = described_class.construct_uri
        expect(uri).to eq(URI.parse("https://es.run.xxx.com/logstash-2014.10.10/_search"))
      end
    
      it 'construct the payload base on the time stamp' do
        allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
        payload = described_class.construct_payload
        pl = JSON.parse(payload)
        expect(pl['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['to']).to eq(1412899200000)
        expect(pl['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['from']).to eq(1412899200000-5*60*1000)
      end
    
      context 'search result' do
    
        before :each do
          @http = double(Net::HTTP)
          allow(@http).to receive(:verify_mode=).with(0)
          @req = double(Net::HTTP::Get)
          @response = double(Net::HTTPResponse)
    
          allow(DateTime).to receive(:now).and_return(DateTime.new(2014,10,10))
          allow(Net::HTTP).to receive(:new).with('es.run.xxx.com',443).and_return(@http)
          allow(Net::HTTP::Get).to receive(:new).with('/logstash-2014.10.10/_search').and_return(@req)
          allow(@http).to receive(:use_ssl=).with(true)
          allow(@req).to receive(:basic_auth).with('fake_log_user','fake_log_pass')
          allow(@req).to receive(:[]=).with('Content-Type','application/json')
    
          allow(@req).to receive(:body=)
    
        end
    
        it 'raise errors when have Net::Http exceptions' do
          allow(@http).to receive(:request).with(@req).and_raise(Timeout::Error)
          allow(STDOUT).to receive(:puts).with("[ERR] CAN NOT GET Logstash logs, Timeout::Error")
          result = described_class.search_result
          expect(result).to eq({})
        end
    
        it 'grab search result and return a hash with error instance ip:port' do
    
          simulate_response_body = '
        {
          "took" : 19,
          "timed_out" : false,
          "_shards" : {
            "total" : 6,
            "successful" : 6,
            "failed" : 0
          },
          "hits" : {
            "total" : 1,
            "max_score" : null,
            "hits" : [ {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null,
              "_source" : {"@message":"2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read tcp 10.10.17.37:63426: i/o timeout\n","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
              "sort" : [ 1413878323092 ]
            },
            {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null,
              "_source" : {"@message":"2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read empty string to test","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
              "sort" : [ 1413878323092 ]
              }
            ]
          }
        }'
    
        allow(@http).to receive(:request).with(@req).and_return(@response)
        allow(@response).to receive(:body).and_return(simulate_response_body)
    
        result = described_class.search_result
        expect(result).to eq({"10.10.17.37:63426"=>1})
        end
    
        it 'returns the search result when message field is missing' do
    
          simulate_response_body = '
        {
          "took" : 19,
          "timed_out" : false,
          "_shards" : {
            "total" : 6,
            "successful" : 6,
            "failed" : 0
          },
          "hits" : {
            "total" : 1,
            "max_score" : null,
            "hits" : [ {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null,
              "_source" : {"@message":"2014-10-21T07:58:43.092792+00:00 10.10.80.16 vcap.gorouter_ctl.stderr [job=router_z2 index=1]  2014/10/21 07:58:43 http: proxy error: read tcp 10.10.17.37:63426: i/o timeout\n","@version":"1","@timestamp":"2014-10-21T07:59:05.425Z","type":"syslog","host":"10.10.66.17","tags":["_grokparsefailure"],"priority":13,"severity":5,"facility":1,"facility_label":"user-level","severity_label":"Notice","job":"router_z2","index":"1","msg_timestamp":"2014-10-21T07:58:43.092792+00:00","origin":"10.10.80.16"},
              "sort" : [ 1413878323092 ]
            },
            {
              "_index" : "logstash-2014.10.21",
              "_type" : "syslog",
              "_id" : "UyuLFsXGSriM-gVfom3-hw",
              "_score" : null, "_source" : {},
              "sort" : [ 1413878323092 ]
              }
            ]
          }
        }'
    
        allow(@http).to receive(:request).with(@req).and_return(@response)
        allow(@response).to receive(:body).and_return(simulate_response_body)
    
        result = described_class.search_result
        expect(result).to eq({"10.10.17.37:63426"=>1})
        end
      end
    
    end

## assert/config_spec.yml

log_search:

url: fake_log_uri

login: fake_log_user

pass: fake_log_pass

timespan: 5

5. 简单rakefile

1. require 'yaml'
   
2. require 'erb'
   
3. 
   
4. 
   
5. ENV['RUBY_ENVIRONMENT'] ||= 'staging'
   
6. 
   
7. OUTPUT_FILE='manifest.yml'
   
8. TEMPLATE_FILE=OUTPUT_FILE+ '.erb'
   
9. 
   
10. 
    
11. def get_template
    
12.   File.read(TEMPLATE_FILE)
    
13. end
    
14. 
    
15. 
    
16. RSpec::Core::RakeTask.new(:spec)
    
17. task default: [:spec]
    
18. 
    
19. 
    
20. desc "push to xxx.com"
    
21. task :push do
    
22.   Rake::Task["login"].invoke
    
23.   puts %x[cf push --no-start]
    
24.   Rake::Task["set-env"].invoke
    
25.   puts %x[cf start app-instance-monitor]
    
26.   Rake::Task["logout"].invoke
    
27. end
    
28. 
    
29. 
    
30. desc "set env of app"
    
31. task :'set-env' do
    
32.   puts system("COMMAND RUBY_ENVIRONMENT #{ENV['RUBY_ENVIRONMENT']}")
    
33. end
    
34. 
    
35. 
    
36. desc "login to xxx.com"
    
37. task :login do
    
38.   conf = YAML::load(File.open('config/config.yml'))[ENV['RUBY_ENVIRONMENT']]
    
39.   puts %x[COMMAND #{conf['xxxx']['api']} --skip-ssl-validation]
    
40.   puts %x[cf login -u #{conf['xxxx']['username']} -p #{conf['user']['password']} -o system -s monitor]
    
41. end
    
42. 
    
43. 
    
44. desc "logout of xxx.com"
    
45. task :logout do
    
46.   puts %x[cf logout]
    
47. end

 6.  关于 Gemfile
source 'https://ruby.taobao.org'   #国外的就是rubygems.org，但国内用这个太慢了。
gem 'rake'  '~> xxx'   指定gem 的版本

7.    .ruby-version
2.2.4

8. 获取AWS的使用情况，代码练习。

1. #!/usr/bin/env ruby
   
2. 
   
3. require 'optparse'
   
4. require 'aws'
   
5. 
   
6. class AwsInfoHandler
   
7.   def self.acquire_account_info(account)
   
8.     name=account.fetch('name')
   
9.     id=account.fetch('account_id')
   
10. 
    
11. 
    
12.     ec2 = AWS::EC2.new(
    
13.       :access_key_id => account.fetch('aws_key'),
    
14.       :secret_access_key => account.fetch('aws_sec'),
    
15.       :region => account.fetch('region')
    
16.     )
    
17. 
    
18.     ec2.instances.inject({}) do |m, i|
    
19.       if m[i.availability_zone].nil?
    
20.         m[i.availability_zone] = {}
    
21.       end
    
22. 
    
23.       if m[i.availability_zone][i.instance_type].nil?
    
24.         m[i.availability_zone][i.instance_type]=1
    
25.       else
    
26.         m[i.availability_zone][i.instance_type]+=1
    
27.       end
    
28.       m
    
29.     end
    
30.   end
    
31. 
    
32. 
    
33. #### reservation, for S3 ###
    
34. 
    
35. 
    
36.   def self.publish_s3_report(s3_input_config)
    
37.     account = s3_input_config[:account]
    
38.     s3 = AWS::S3.new(
    
39.       :access_key_id => account.fetch('aws_key'),
    
40.       :secret_access_key => account.fetch('aws_sec')
    
41.     )
    
42.     bucket = s3.buckets[s3_input_config[:target_bucket]]
    
43.     object = bucket.objects[s3_input_config[:target_filename]]
    
44.     object.write(s3_input_config[:contents])
    
45.   end
    
46. end
    
47. 
    
48. 
    
49. ######################
    
50. 
    
51. 
    
52. class ReportGenerator
    
53.   def initialize(accounts)
    
54.     @accounts = accounts
    
55.   end
    
56. 
    
57. 
    
58.   def generate_report
    
59.     report=[]
    
60. 
    
61. 
    
62.     @accounts.each do |ac|
    
63.       info = AwsInfoHandler.acquire_account_info(ac)
    
64.       puts "[INFO] Generating account info for #{ac.fetch('name')}"
    
65.       info.each do |k,v|
    
66.         v.each do |type, count|
    
67.           rp = {
    
68.             'account_name' => ac.fetch('name'),
    
69.             'account_id' => ac.fetch('account_id'),
    
70.             'AZ' => k,
    
71.             'flavor' => type,
    
72.             'count' => count
    
73.           }
    
74.           report << rp
    
75.         end
    
76.       end
    
77.     end
    
78.     report
    
79.   end
    
80. end
    
81. 
    
82. 
    
83. class EmailAgent
    
84.   def self.generate_and_email(report,report_to=nil)
    
85.     puts "[DEBUG] report for #{Time.now.strftime("%m/%d/%Y")}"
    
86.     puts "=========================="
    
87.     puts report
    
88.     puts "=========================="
    
89.     puts "[DEBUG] End of report"
    
90. 
    
91. 
    
92. =begin
    
93. MAIL_CONF={"mail-provider":[
    
94.    {
    
95.            "credentials": {
    
96.            "password": "qdWC4sqfLoD9Zg",
    
97.            "username": "cloudfoundry"
    
98.           }
    
99.     ]
    
100.    }
     
101. }
     
102. =end
     
103.     credentials = host = username = password = ''
     
104.     if !ENV['MAIL_CONF'].to_s.empty?
     
105.       JSON.parse(ENV['MAIL_CONF']).each do |k,v|
     
106.         if !k.scan("mail-provider").to_s.empty?
     
107.           credentials = v.first.select {|k1,v1| k1 == "credentials"}["credentials"]
     
108.           host = credentials["hostname"]
     
109.           username = credentials["username"]
     
110.           password = credentials["password"]
     
111.         end
     
112.         end
     
113.     end
     
114. 
     
115. 
     
116. ### SMTP setings ###
     
117. # ......
     
118. 
     
119. 
     
120. # attachement
     
121. 
     
122. 
     
123.       report_file = File.new('/tmp/report.csv', 'w+')
     
124.       report_file.puts(report[0].keys.join(','))
     
125. 
     
126. 
     
127.       report.each do |r|
     
128.         report_file.puts(r.values.join(','))
     
129.       end
     
130. 
     
131. 
     
132.       report_file.close
     
133. ### send out ###
     
134. ## ......
     
135.   end
     
136. end
     
137. 
     
138. ###  -c 来指定配置文件，实现代码和配置分离。
     
139. options = {}
     
140. optparse = OptionParser.new do |opts|
     
141.   opts.on('-h', '--help', 'Help Messages') do
     
142.     puts opts
     
143.     exit
     
144.   end
     
145.   opts.on('-c', '--config File', 'The Config file') do |f|
     
146.     options[:file] = f
     
147.   end
     
148. end
     
149. optparse.
     
150. unless options[:file]
     
151.   puts 'no config file specified, exit'
     
152.   exit
     
153. end
     
154. 
     
155. 
     
156. ruby_env = ENV['PLATFORM'] || 'production'
     
157. conf = YAML::load(File.open(options[:file]))[ruby_env]
     
158. 
     
159. 
     
160. accounts = conf.fetch('accounts')
     
161. mail_to = conf.fetch('to')
     
162. report = ReportGenerator.new(accounts).generate_report
     
163. EmailAgent.generate_and_email(report, mail_to)

# config.yml

1. ---
   production:
     accounts:
     - name: web
       account_id: xxxxxxxx7842
       aws_key: xxxx
       aws_sec: xxxx
     - name: db
       account_id: xxxxxxxx6344
       aws_key: xxx
       aws_sec: xxxx
   
     reports_email_to:
     - [email protected]