ELK-WEB日志分析实战

ELK日志分析平台

ELK架构图例

css 复制代码

Elasticsearch
Logstash
web cluster
es-0001
es-0002
es-0003
es-0004
es-0005
output
filter
input
filebeat
apache
filebeat
apache
filebeat
apache
kibana

logstash安装

购买云主机

主机	IP地址	配置
logstash	192.168.1.27	最低配置4核8G

安装logstash

csharp 复制代码

[root@logstash ~]# vim /etc/hosts
192.168.1.21    es-0001
192.168.1.22    es-0002
192.168.1.23    es-0003
192.168.1.24    es-0004
192.168.1.25    es-0005
192.168.1.27    logstash
[root@logstash ~]# yum install -y java-1.8.0-openjdk-devel logstash

基础配置样例

csharp 复制代码

[root@logstash ~]# ln -s /etc/logstash /usr/share/logstash/config
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input { 
  stdin {}
}

filter{ }

output{ 
  stdout{}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash

插件与调试格式

使用json格式字符串测试 {"a":"1", "b":"2", "c":"3"}

ini 复制代码

[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input { 
  stdin { codec => "json" }
}

filter{ }

output{ 
  stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash

官方手册地址

input file插件

ini 复制代码

[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
  file {
    path => ["/tmp/c.log"]
    type => "test"
    start_position => "beginning"
    sincedb_path => "/var/lib/logstash/sincedb"
  }
}
filter{ }
output{ 
  stdout{ codec => "rubydebug" }
}
[root@logstash ~]# rm -rf /var/lib/logstash/plugins/inputs/file/.sincedb_*
[root@logstash ~]# /usr/share/logstash/bin/logstash

filter grok插件

正则表达式分组匹配格式: (?<名字>正则表达式) 正则表达式宏调用格式: %{宏名称:名字} 宏文件路径 : /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.1.2/patterns

ini 复制代码

[root@logstash ~]# echo '192.168.1.252 - - [29/Jul/2020:14:06:57 +0800] "GET /info.html HTTP/1.1" 200 119 "-" "curl/7.29.0"' >/tmp/c.log
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {
  file {
    path => ["/tmp/c.log"]
    type => "apache_log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}
filter{ 
  grok {
    match => { "message" => "%{HTTPD_COMBINEDLOG}" }
    remove_field => ["message"]
  }
}
output{ 
  stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash

output elasticsearch插件

ini 复制代码

[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input { 
  file {
    path => ["/tmp/c.log"]
    type => "test"
    start_position => "beginning"
    sincedb_path => "/var/lib/logstash/sincedb"
  }
}

filter{
  grok {
    match => { "message" => "%{HTTPD_COMBINEDLOG}" }
    remove_field => ["message"]
  }
}

output{ 
  stdout{ codec => "rubydebug" }
  elasticsearch {
    hosts => ["es-0004:9200", "es-0005:9200"]
    index => "weblog-%{+YYYY.MM.dd}"
  }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash

浏览器打开 head 插件，通过 web 页面浏览验证

WEB日志分析实战

beats配置

ini 复制代码

[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input { 
  beats {
    port => 5044
  }
} 

filter{
  grok {
    match => { "message" => "%{HTTPD_COMBINEDLOG}" }
    remove_field => ["message"]
  }
}

output{ 
  stdout{ codec => "rubydebug" }
  elasticsearch {
    hosts => ["es-0004:9200", "es-0005:9200"]
    index => "weblog-%{+YYYY.MM.dd}"
  }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash

filebeat安装配置

yaml 复制代码

[root@web ~]# yum install -y filebeat
[root@web ~]# systemctl enable filebeat
[root@web ~]# vim /etc/filebeat/filebeat.yml
24:  enabled: true # 打开收集模块
28:  - /var/log/httpd/access_log # 定义日志路径
148: # 注释掉
150: # 注释掉
161: output.logstash: # 设置输出模块
163: hosts: ["192.168.1.27:5044"] # 输出给logstash
179: # 收集系统相关信息，可以注释掉
180: # 收集系统相关信息，可以注释掉
181: # 收集系统相关信息，可以注释掉
[root@web ~]# rm -f /var/log/httpd/*
[root@web ~]# systemctl restart filebeat httpd

自定义日志标签

filebeat配置

yaml 复制代码

[root@web ~]# vim /etc/filebeat/filebeat.yml
45:  fields:
46:    logtype: apache_log
[root@web ~]# grep -Pv "^\s*(#|$)" /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/httpd/access_log
  fields:
    logtype: apache_log
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
output.logstash:
  hosts: ["192.168.1.27:5044"]
[root@web ~]# rm -f /var/log/httpd/*
[root@web ~]# systemctl restart filebeat httpd

logstash配置

ini 复制代码

[root@logstash ~]# cat /etc/logstash/conf.d/my.conf
input { 
  beats {
    port => 5044
  }
}

filter{
  if [fields][logtype] == "apache_log" {
  grok {
    match => { "message" => "%{HTTPD_COMBINEDLOG}" }
    remove_field => ["message"]
  }}
}

output{ 
  stdout{ codec => "rubydebug" }
  if [fields][logtype] == "apache_log" {
  elasticsearch {
    hosts => ["es-0004:9200", "es-0005:9200"]
    index => "weblog-%{+YYYY.MM.dd}"
  }}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash