1、原始数据源服务器,从日志拉取数据,并转化为utf8编码,增加ip地址,保存到Redis中,上配置如下:
- input {
- file {
- path => "/tmp/123server.log"
- codec => json {
- charset => "GBK"
- }
- start_position => "beginning"
- sincedb_path => "/dev/null"
- type => "activitysun"
- }
- }
- filter {
- date {
- match => ["timestamp", "UNIX"]
- remove_field => ["time"]
- }
- #ruby {
- # code => "event.timestamp.time.localtime"
- #}
- mutate {
- #some pc no host
- replace => { "host" => "192.168.10.11" }
- }
- }
- output {
- #stdout {
- # codec => plain {
- # charset => "UTF-8"
- # }
- #}
- file {
- path => "/tmp/logstash.log"
- codec => json {
- charset => "UTF-8"
- }
- }
- redis {
- host => ["192.168.10.18"]
- port => 16378
- data_type => "list"
- key => "123server"
- codec => json {
- charset => "UTF-8"
- }
- }
- }
采用如下命令启动
- /data/logstash-2.3.4/bin/logstash -f activitylog.conf
- input {
- redis {
- host => ["192.168.10.18"]
- port => 26378
- data_type => "list"
- key => "123server"
- codec => json {
- charset => "UTF-8"
- }
- #type => "activitysun"
- }
- }
- filter {
- ruby {
- code=>"event['daytag']=event.timestamp.time.localtime.strftime('%Y.%m.%d')"
- }
- }
- output {
- elasticsearch {
- hosts => ["127.0.0.1:19200"]
- index => "%{type}-%{daytag}"
- #index => "%{type}-%{+yyyy.MM.dd}"
- }
- }
filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /log/123.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: pchannelserver
fields_under_root: true
fields:
host: 192.168.10.15
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.18"]
password: "8841c09BAD52E63067C4DA"
port: 15377
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: true
path: "/tmp/filebeat"
output.console:
enabled: false
- input_type: log
paths:
- /log/123.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: pchannelserver
fields_under_root: true
fields:
host: 192.168.10.15
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.18"]
password: "8841c09BAD52E63067C4DA"
port: 15377
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: true
path: "/tmp/filebeat"
output.console:
enabled: false