nginx收集指定接口日志到elk

配置filebeat

grep -Ev "^ #|$|#|^ #" /data/filebeat/filebeat.yml

yml 复制代码
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data/nginx_logs/nginx-access-*.log
  
  fields:                                       #在日志中增加一个字段,字段为log_topic,值为nginx_access,logstash根据带有这个字段的日志存储到指定的es索引库
    app_name: nginx-appname
    profiles_active: pro
    app_node: nginx_hostname
  fields_under_root: true
  tail_files: true
  include_lines: ['/apis/order/save'] #只收集日志中的指定行
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~
output.kafka:                                   #输出到kafka系统
  enabled: true
  hosts: ["kafka1:9092","kafka2:9092","kafka3:9092"]                           #kafka的地址
  topic: 'nginx_appname_topic'               #指定将日志存储到kafka集群的哪个topic中,这里的topic值是引用在inputs中定义的fields,通过这种方式可以将不同路径的日志分别存储到不同的topic中
  username: kafka_user
  password: kafka_password

配置logstash

conf 复制代码
 cat /usr/local/app/logstash/config/logstash.conf 
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
        kafka {                         #类型为kafka
                bootstrap_servers => ["kafka1:9092,kafka2:9092,kafka3:9092"]                    #kafka集群地址
                group_id => 'logstash_groupname_consumer'
                topics => ["pro_log_topic","test_log_topic","uat_log_topic","nginx_appname_topic"]                     #要读取那些kafka topics
                client_id => "appname_pro_logs"
                consumer_threads => 3
                sasl_mechanism => "PLAIN"
                security_protocol => "SASL_PLAINTEXT"
                sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka_user'  password='kafka_password';"
                codec => "json"                                                                         #处理json格式的数据
                auto_offset_reset => "latest"                                           #只消费最新的kafka数据
        }
        kafka {                         #类型为kafka
                bootstrap_servers => ["kafkaip:9092"]                        #kafka集群地址
                group_id => 'logstash_groupname_consumer2'
                topics => ["topic"]                 #要读取那些kafka topics
                client_id => "appname_test_logs"
                consumer_threads => 3
                sasl_mechanism => "PLAIN"
                security_protocol => "SASL_PLAINTEXT"
                sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka_user'  password='kafka_password';"
                codec => "json"                                                                         #处理json格式的数据
                auto_offset_reset => "latest"                                           #只消费最新的kafka数据
        }


}

filter {
        mutate {
                lowercase => ["app_name"]
                remove_field => ["_index","_id","_type","_version","_score","referer","agent","@version"]                 #删除没用的字段
        }
        date {
        match => ["date", "yyyy-MM-dd HH:mm:ss.SSS"]
                target => '@timestamp'
                timezone => 'Asia/Shanghai'
        }
        ruby{
                code => "event.set('index_day', (event.get('@timestamp').time.localtime).strftime('%Y.%m.%d'))"
        }
}


output {
  elasticsearch {
    hosts => ["172.19.189.179:9200","172.19.38.38:9200","172.19.38.39:9200"]
    index => "%{[app_name]}-%{[profiles_active]}-%{index_day}"
    #index => "%{[app_name]}-%{[profiles_active]}-%{+YYYY.MM.dd}"
    codec =>  "json"
    user => "elastic"
    password => "esappname0227"
  }
}
相关推荐
2301_787328491 小时前
25.负载均衡-Nginx、HAProxy、LVS 全解析
nginx·负载均衡·lvs
siriuuus1 小时前
Nginx 负载均衡调度算法
运维·nginx·负载均衡
tjsoft9 小时前
设置 windows nginx.exe 每天 重启
运维·windows·nginx
舰长11510 小时前
nginx 负载均衡配置
运维·nginx·负载均衡
foundbug9991 天前
查看nginx日志文件
linux·nginx·github
一只程序烽.1 天前
java项目使用宝塔面板部署服务器nginx不能反向代理找到图片资源
java·服务器·nginx
福大大架构师每日一题1 天前
nginx 1.29.2 发布:AWS-LC 支持、QUIC及SSL优化等重要更新
nginx·ssl·aws
金色天际线-1 天前
nginx + spring cloud + redis + mysql + ELFK 部署
redis·nginx·spring cloud
帅帅梓2 天前
nginx访问控制 用户认证 https
运维·nginx·https
有谁看见我的剑了?2 天前
k8s ingress-nginx 学习
学习·nginx·kubernetes