ELK+Kafka+zookeeper+Filebeat(nginx)
less
------------------------------ ELK+Kafka+zookeeper+Filebeat(nginx) --------------------------------
192.168.11.150 es1 Elasticsearch+Elasticsearch-head+phantomjs+node
192.168.11.151 es2 Elasticsearch+Elasticsearch-head+phantomjs+node
192.168.11.152 logstash+kibana
192.168.11.144 zookeeper+kafka
192.168.11.145 zookeeper+kafka
192.168.11.146 zookeeper+kafka
192.168.11.137 filebeat+nginx
less
192.168.11.137 filebeat+nginx
[root@test2 ~]# cd /opt/
[root@test2 ~]# tar -xf filebeat-6.7.2-linux-x86_64.tar.gz
[root@test2 ~]# mv filebeat-6.7.2-linux-x86_64 filebeat
[root@test2 ~]# cd filebeat/
[root@test2 filebeat]# pwd
/opt/filebeat
[root@test2 filebeat]# vim filebeat.yml
filebeat.prospectors:
- type: log
enabled: true
paths:
- /var/log/nginx/access_log
tags: ["access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error_log
tags: ["error"]
#添加输出到 Kafka 的配置
161行
output.kafka:
enabled: true
hosts: ["192.168.11.144:9092","192.168.11.145:9092","192.168.11.146:9092"] #指定 Kafka 集群配置
topic: "nginx" #指定 Kafka 的 topic
#启动 filebeat
nohup ./filebeat -e -c filebeat.yml> filebeat.out &
less
192.168.11.152
[root@apache-elk3 ~]# cd /etc/logstash/conf.d/
[root@apache-elk3 conf.d]# vim kafka.conf
input {
kafka {
bootstrap_servers => "192.168.11.144:9092","192.168.11.145:9092","192.168.11.146:9092"
#kafka集群地址
topics => "nginx"
#拉取的kafka的指定topic
type => "nginx_kafka"
#指定 type 字段
codec => "json"
#解析json格式的日志数据
auto_offset_reset => "latest"
#拉取最近数据,earliest为从头开始拉取
decorate_events => true
#传递给elasticsearch的数据额外增加kafka的属性数据
}
}
output {
if "access" in [tags] {
elasticsearch {
hosts => ["192.168.11.150:9200","192.168.11.151:9200"]
index => "nginx_access-%{+YYYY.MM.dd}"
}
}
if "error" in [tags] {
elasticsearch {
hosts => ["192.168.11.150:9200","192.168.11.151:9200"]
index => "nginx_error-%{+YYYY.MM.dd}"
}
}
}
#启动 logstash
logstash -f kafka.conf --path.data /opt/test9 &
4.浏览器访问 http://192.168.39.60:5601 登录 Kibana,
单击"Create Index Pattern"按钮添加索引"filebeat_test-*",单击 "create" 按钮创建,
单击 "Discover" 按钮可查看图表信息及日志信息。
x Pattern"按钮添加索引"filebeat_test-*",单击 "create" 按钮创建,
单击 "Discover" 按钮可查看图表信息及日志信息。