elk + filebeat 采集json日志

elk下载安装：

https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.3.2-linux-x86_64.tar.gz

filebeat-配置文件：

filebeat.inputs:
- type: log
  paths:
    - /xxx/logs/*.log
  tags: ["xxx_log", "iperf3_host"]
  json.message_key: levelname
  include_lines: ['info']
  json.add_error_key: true
  json.keys_under_root: true
output.logstash:
  hosts: ["xxx:<port>"]
#output.elasticsearch:
#  hosts: ["xxx:<port>"]
#  index: "xxx-host-iperfs-index-%{+YYYY.MM.dd}"
setup.template.name: "filebeattest"
setup.template.pattern: "filebeattest-*"

logstash-配置文件：

[root@xxx conf.d]# cat xxx-log-pipeline.conf 
##################################
# 测试执行命令：
# /usr/share/logstash/bin/logstash -f xxx-log-pipeline.conf --config.reload.automatic
##################################

input {
    beats {
        port => xxx
    }
}

filter {
    if "xxx_log" in [tags] and [tags][1] {
        mutate {
            add_field => { "log_project_name"  => "%{[tags][1]}" }
        }
    }
    if [tags][1] == "xxx-nginx" {
        mutate {
            convert => [ "status","integer" ]
            convert => [ "size","integer" ]
            convert => [ "upstreamtime","float" ]
            remove_field => "message"
        }
        geoip {
            source => "ip"
            target => "ipgeo"
        }
    }
}

output {
    if "xxx_log" in [tags] and [log_project_name] {
              # stdout { codec => rubydebug }
        elasticsearch {
            hosts => ["xxx:<port>"]
            index => "xxx-log-%{log_project_name}-%{+YYYY.MM.dd}"
        }
    }
}


[root@xxx logstash]# 
[root@xxx logstash]# cat pipelines.yml 
# This file is where you define your pipelines. You can define multiple.
# For more information on multiple pipelines, see the documentation:
#   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html

- pipeline.id: main
  path.config: "/etc/logstash/conf.d/*.conf"


[root@xxx logstash]# cat logstash.yml | egrep -v '^#|^$|#'
path.data: /var/lib/logstash
pipeline.ecs_compatibility: disabled
path.logs: /var/log/logstash

es-配置：

[root@xxx elasticsearch]# cat elasticsearch.yml | egrep -v '^#|^$|#'
cluster.name: log-es-production-new-1
node.name: xxx
node.roles: [ "master", "data" ]
xpack.security.enabled: false
xpack.security.transport.ssl.enabled: false
xpack.security.http.ssl.enabled: false
ingest.geoip.downloader.enabled: false
path.data: [/es_data1/production/elasticsearch/lib,/es_data2/production/elasticsearch/lib]
path.logs: /home/elasticsearch/production/elasticsearch/log
bootstrap.memory_lock: true
network.host: 172.26.2.88
transport.port: 39402
http.port: 39202
discovery.seed_hosts: [xxx, xxx, xxx, xxx, xxx]
cluster.initial_master_nodes: [xxx, xxx, xxx, xxx, xxx]
action.destructive_requires_name: true
action.auto_create_index: true
indices.fielddata.cache.size: 3%
cluster.routing.allocation.disk.threshold_enabled: true
cluster.routing.allocation.disk.watermark.low: 150g
cluster.routing.allocation.disk.watermark.high: 50g
cluster.routing.allocation.disk.watermark.flood_stage: 40g

kinaba就不多介绍了，安装配置即可。

总结：

elk + filebeat 的采集日志系统框架是比较常规的一种模式， 缺陷是使用logstash是比较耗费资源的，如果量大的情况下，建议与开发商讨，直接把日志打进kafka，之后output到filebeat进行消费，最后output到es，到kibana展示。

如果懒得开发，也可以直接使用云展示，直接把数据output到云上的elk，也是一种选择。