Filebeat官方文档地址
https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation.html
下载和安装
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.3.1-linux-x86_64.tar.gz
tar xzvf filebeat-7.3.1-linux-x86_64.tar.gz
编写filebeat.yml
启动
chmod go-w /home/lintong/software/apache/filebeat-7.3.1-linux-x86_64/filebeat.yml
./filebeat -e -c filebeat.yml
codec.format
codec.format:
string: '%{[@timestamp]} %{[message]}'
输出
2019-09-13T17:06:51.797Z 123123123123
codec.json
codec.json:
pretty: true
escape_html: false
输出
{
"@timestamp": "2019-09-13T09:08:49.590Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.3.1",
"topic": "thrift_json_source"
},
"host": {
"name": "lintong-B250M-DS3H"
},
"agent": {
"version": "7.3.1",
"type": "filebeat",
"ephemeral_id": "60b93a10-dcce-499b-ae81-0755bfc8bf5c",
"hostname": "lintong-B250M-DS3H",
"id": "6ebb0912-ffce-4ddd-9cc8-7bf624e62c78"
},
"ecs": {
"version": "1.0.1"
},
"message": "123123123123",
"log": {
"file": {
"path": "/home/lintong/下载/test.log"
},
"offset": 0
},
"input": {
"type": "log"
}
}
如果pretty是false将输出
{"@timestamp":"2019-09-13T09:10:50.164Z","@metadata":{"beat":"filebeat","type":"_doc","version":"7.3.1","topic":"thrift_json_source"},"log":{"file":{"path":"/home/lintong/下载/test.log"},"offset":0},"message":"123123123123","input":{"type":"log"},"ecs":{"version":"1.0.1"},"host":{"name":"lintong-B250M-DS3H"},"agent":{"type":"filebeat","ephemeral_id":"b26075f9-37f8-4d95-9341-fccc4504c1b5","hostname":"lintong-B250M-DS3H","id":"6ebb0912-ffce-4ddd-9cc8-7bf624e62c78","version":"7.3.1"}}
如果需要添加字段
paths:
- /home/lintong/下载/test.log
fields:
add_field: lintong
输出
{
"@timestamp": "2019-09-16T08:16:06.169Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.3.1",
"topic": "thrift_json_source"
},
"host": {
"name": "lintong-B250M-DS3H"
},
"log": {
"offset": 31,
"file": {
"path": "/home/lintong/下载/test.log"
}
},
"message": "33333333",
"input": {
"type": "log"
},
"fields": {
"add_field": "lintong"
},
"agent": {
"type": "filebeat",
"ephemeral_id": "c16102da-421d-4ff3-90ad-1737451a909d",
"hostname": "lintong-B250M-DS3H",
"id": "6ebb0912-ffce-4ddd-9cc8-7bf624e62c78",
"version": "7.3.1"
},
"ecs": {
"version": "1.0.1"
}
}
在codec.format中添加字段
codec.format:
string: '%{[@timestamp]} %{[fields.add_field]} %{[message]}'
输出
2019-09-16T16:18:34.048Z lintong 55555555
如果想添加的字段在json的顶层，就是不在fields字段下层
paths:
- /home/lintong/下载/test.log
fields:
add_field: lintong
fields_under_root: true
输出
{
"@timestamp": "2019-09-16T08:22:43.997Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.3.1",
"topic": "thrift_json_source"
},
"agent": {
"ephemeral_id": "d8e45d90-6434-4e0d-a6fc-74611b87cbd4",
"hostname": "lintong-B250M-DS3H",
"id": "6ebb0912-ffce-4ddd-9cc8-7bf624e62c78",
"version": "7.3.1",
"type": "filebeat"
},
"log": {
"offset": 58,
"file": {
"path": "/home/lintong/下载/test.log"
}
},
"message": "66666666",
"add_field": "lintong",
"input": {
"type": "log"
},
"ecs": {
"version": "1.0.1"
},
"host": {
"name": "lintong-B250M-DS3H"
}
}
如果要去掉不要的字段
参考：
https://studygolang.com/articles/10935
和
https://www.elastic.co/guide/en/beats/filebeat/current/drop-fields.html
比如
processors:
- drop_fields:
fields: ["host", "log", "input","ecs","agent"]
输出
{
"@timestamp": "2019-09-16T08:55:55.934Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.3.1",
"topic": "thrift_json_source"
},
"message": "33333333",
"add_field": "lintong"
}
其中@metadata和@timestamp不能在filebeat中去掉