nginx收集指定接口日志到elk

配置filebeat

grep -Ev "^ #|$|#|^ #" /data/filebeat/filebeat.yml

yml 复制代码
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data/nginx_logs/nginx-access-*.log
  
  fields:                                       #在日志中增加一个字段,字段为log_topic,值为nginx_access,logstash根据带有这个字段的日志存储到指定的es索引库
    app_name: nginx-appname
    profiles_active: pro
    app_node: nginx_hostname
  fields_under_root: true
  tail_files: true
  include_lines: ['/apis/order/save'] #只收集日志中的指定行
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~
output.kafka:                                   #输出到kafka系统
  enabled: true
  hosts: ["kafka1:9092","kafka2:9092","kafka3:9092"]                           #kafka的地址
  topic: 'nginx_appname_topic'               #指定将日志存储到kafka集群的哪个topic中,这里的topic值是引用在inputs中定义的fields,通过这种方式可以将不同路径的日志分别存储到不同的topic中
  username: kafka_user
  password: kafka_password

配置logstash

conf 复制代码
 cat /usr/local/app/logstash/config/logstash.conf 
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
        kafka {                         #类型为kafka
                bootstrap_servers => ["kafka1:9092,kafka2:9092,kafka3:9092"]                    #kafka集群地址
                group_id => 'logstash_groupname_consumer'
                topics => ["pro_log_topic","test_log_topic","uat_log_topic","nginx_appname_topic"]                     #要读取那些kafka topics
                client_id => "appname_pro_logs"
                consumer_threads => 3
                sasl_mechanism => "PLAIN"
                security_protocol => "SASL_PLAINTEXT"
                sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka_user'  password='kafka_password';"
                codec => "json"                                                                         #处理json格式的数据
                auto_offset_reset => "latest"                                           #只消费最新的kafka数据
        }
        kafka {                         #类型为kafka
                bootstrap_servers => ["kafkaip:9092"]                        #kafka集群地址
                group_id => 'logstash_groupname_consumer2'
                topics => ["topic"]                 #要读取那些kafka topics
                client_id => "appname_test_logs"
                consumer_threads => 3
                sasl_mechanism => "PLAIN"
                security_protocol => "SASL_PLAINTEXT"
                sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka_user'  password='kafka_password';"
                codec => "json"                                                                         #处理json格式的数据
                auto_offset_reset => "latest"                                           #只消费最新的kafka数据
        }


}

filter {
        mutate {
                lowercase => ["app_name"]
                remove_field => ["_index","_id","_type","_version","_score","referer","agent","@version"]                 #删除没用的字段
        }
        date {
        match => ["date", "yyyy-MM-dd HH:mm:ss.SSS"]
                target => '@timestamp'
                timezone => 'Asia/Shanghai'
        }
        ruby{
                code => "event.set('index_day', (event.get('@timestamp').time.localtime).strftime('%Y.%m.%d'))"
        }
}


output {
  elasticsearch {
    hosts => ["172.19.189.179:9200","172.19.38.38:9200","172.19.38.39:9200"]
    index => "%{[app_name]}-%{[profiles_active]}-%{index_day}"
    #index => "%{[app_name]}-%{[profiles_active]}-%{+YYYY.MM.dd}"
    codec =>  "json"
    user => "elastic"
    password => "esappname0227"
  }
}
相关推荐
异常君2 小时前
HTTP头中的Accept-Encoding与Content-Encoding深度剖析
后端·nginx·http
生命有所坚持而生存可以随遇而安7 小时前
https nginx 负载均衡配置
服务器·nginx·负载均衡
Python私教8 小时前
CentOS 7 基于 Nginx 的 HTML 部署全流程指南
nginx·centos·html
五号厂房1 天前
客户端收到413 Request Entity Too Large错误该如何解决?
nginx
大胆刁民1 天前
nginx
nginx
王者鳜錸1 天前
2024从Maven-MySQL-Nginx部署
mysql·nginx·maven
中国lanwp1 天前
Pingora vs. Nginx vs. 其他主流代理服务器性能对比
运维·nginx
李菠菜1 天前
利用Nginx实现高性能的前端打点采集服务(支持GET和POST)
linux·前端·nginx
我怎么能这么帅气1 天前
为什么你的录音API在测试环境突然消失?程序员必看的Nginx局域网HTTPS求生指南
javascript·nginx
佳腾_2 天前
【web服务_负载均衡Nginx】二、Nginx 核心技术之负载均衡与反向代理
前端·nginx·云计算·负载均衡·web中间件