nginx收集指定接口日志到elk

配置filebeat

grep -Ev "^ #|$|#|^ #" /data/filebeat/filebeat.yml

yml 复制代码
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data/nginx_logs/nginx-access-*.log
  
  fields:                                       #在日志中增加一个字段,字段为log_topic,值为nginx_access,logstash根据带有这个字段的日志存储到指定的es索引库
    app_name: nginx-appname
    profiles_active: pro
    app_node: nginx_hostname
  fields_under_root: true
  tail_files: true
  include_lines: ['/apis/order/save'] #只收集日志中的指定行
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~
output.kafka:                                   #输出到kafka系统
  enabled: true
  hosts: ["kafka1:9092","kafka2:9092","kafka3:9092"]                           #kafka的地址
  topic: 'nginx_appname_topic'               #指定将日志存储到kafka集群的哪个topic中,这里的topic值是引用在inputs中定义的fields,通过这种方式可以将不同路径的日志分别存储到不同的topic中
  username: kafka_user
  password: kafka_password

配置logstash

conf 复制代码
 cat /usr/local/app/logstash/config/logstash.conf 
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
        kafka {                         #类型为kafka
                bootstrap_servers => ["kafka1:9092,kafka2:9092,kafka3:9092"]                    #kafka集群地址
                group_id => 'logstash_groupname_consumer'
                topics => ["pro_log_topic","test_log_topic","uat_log_topic","nginx_appname_topic"]                     #要读取那些kafka topics
                client_id => "appname_pro_logs"
                consumer_threads => 3
                sasl_mechanism => "PLAIN"
                security_protocol => "SASL_PLAINTEXT"
                sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka_user'  password='kafka_password';"
                codec => "json"                                                                         #处理json格式的数据
                auto_offset_reset => "latest"                                           #只消费最新的kafka数据
        }
        kafka {                         #类型为kafka
                bootstrap_servers => ["kafkaip:9092"]                        #kafka集群地址
                group_id => 'logstash_groupname_consumer2'
                topics => ["topic"]                 #要读取那些kafka topics
                client_id => "appname_test_logs"
                consumer_threads => 3
                sasl_mechanism => "PLAIN"
                security_protocol => "SASL_PLAINTEXT"
                sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka_user'  password='kafka_password';"
                codec => "json"                                                                         #处理json格式的数据
                auto_offset_reset => "latest"                                           #只消费最新的kafka数据
        }


}

filter {
        mutate {
                lowercase => ["app_name"]
                remove_field => ["_index","_id","_type","_version","_score","referer","agent","@version"]                 #删除没用的字段
        }
        date {
        match => ["date", "yyyy-MM-dd HH:mm:ss.SSS"]
                target => '@timestamp'
                timezone => 'Asia/Shanghai'
        }
        ruby{
                code => "event.set('index_day', (event.get('@timestamp').time.localtime).strftime('%Y.%m.%d'))"
        }
}


output {
  elasticsearch {
    hosts => ["172.19.189.179:9200","172.19.38.38:9200","172.19.38.39:9200"]
    index => "%{[app_name]}-%{[profiles_active]}-%{index_day}"
    #index => "%{[app_name]}-%{[profiles_active]}-%{+YYYY.MM.dd}"
    codec =>  "json"
    user => "elastic"
    password => "esappname0227"
  }
}
相关推荐
xujiangyan_5 小时前
nginx的反向代理和负载均衡
服务器·网络·nginx
viqecel19 小时前
网站改版html页面 NGINX 借用伪静态和PHP脚本 实现301重定向跳转
nginx·php·nginx重定向·301重定向·html页面重定向
硪就是硪1 天前
内网环境将nginx的http改完https访问
nginx·http·https
ak啊1 天前
Nginx 安全加固详细配置指南
nginx
沐土Arvin1 天前
Nginx 核心配置详解与性能优化最佳实践
运维·开发语言·前端·nginx·性能优化
haoranyyy2 天前
mac环境中Nginx安装使用 反向代理
linux·服务器·nginx
ak啊2 天前
Nginx 高级缓存配置与优化
nginx
再学一丢丢2 天前
Keepalived+LVS+nginx高可用架构
nginx·架构·lvs
xujiangyan_2 天前
nginx的自动跳转https
服务器·nginx·https
Long_poem2 天前
【自学笔记】ELK基础知识点总览-持续更新
笔记·elk·jenkins