Nginx + ELK + Grafana 全球访问热力图

参考 AKA ES Nginx Logs, https://grafana.com/grafana/dashboards/11190-es-nginx-logs/

整体技术架构一样, 不过有些许区别

文档是7.x版本, 我的环境是8.x ELK
我的公网IP 取值的字段有区别
ES 8.x字段有些区别, 比如geoip.country_name在8.x 改为了geoip.geo.country_name, 一些geoip.x 增加了一层geoip.geo
Grafana报表有部分个性化调整

信息列表


主机	作用	版本
web001	Nginx、Filebeat	8.12.2
ops	Redis、Logstash	8.12.2
eslog001、002、003	Elasticsearch	8.12.2

架构图

Nginx新增日志格式

YAML 复制代码

# 新增
    log_format aka_logs      '{"@timestamp":"$time_iso8601",'
                             '"host":"$hostname",'
                             '"server_ip":"$server_addr",'
                             '"client_ip":"$remote_addr",'
                             '"xff":"$http_x_forwarded_for",'
                             '"domain":"$host",'
                             '"url":"$uri",'
                             '"referer":"$http_referer",'
                             '"args":"$args",'
                             '"upstreamtime":"$upstream_response_time",'
                             '"responsetime":"$request_time",'
                             '"request_method":"$request_method",'
                             '"status":"$status",'
                             '"upstream_status":"$upstream_status",'
                             '"size":"$body_bytes_sent",'
                             '"request_length":"$request_length",'
                             '"protocol":"$server_protocol",'
                             '"upstreamhost":"$upstream_addr",'
                             '"file_dir":"$request_filename",'
                             '"http_user_agent":"$http_user_agent",'
                             '"platform":"$http_platform"'
                             '}';

Bash 复制代码

# 修改
access_log  /data/logs/nginx/xxxx.xxx.com.access_$year-$month-$day.log aka_logs;

Filebeat 部署

YAML 复制代码

#=========================== Filebeat inputs =============================
filebeat.inputs:
# 收集nginx日志
- type: log
  enabled: true
  paths:
    - /data/logs/nginx/*.log
# 日志时json开启这个
  json.keys_under_root: true
  json.overwrite_keys: true
  json.add_error_key: true
  # exclude_lines: ['^.*Blackbox.*$', '^.*ELB-HealthChecker.*$', '^.*favicon.*$']

#-------------------------- Logstash output ------------------------------
# output.logstash:
#   hosts: ["LogstashIP:5044"]

#-------------------------- Redis output ------------------------------
output.redis:
  hosts: ["redisIP:6379"]   #输出到redis的机器
  password: "密码"
  key: "nginx_logs"   #redis中日志数据的key值ֵ
  db: 0
  timeout: 5

YAML 复制代码

# version: '3.8'

networks:
  filebeat-network:
    driver: bridge

services:
  filebeat:
    image: docker.elastic.co/beats/filebeat:8.12.2
    restart: always
    hostname: "{{.Node.Hostname}}-filebeat"
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /data/logs/nginx:/data/logs/nginx
    networks:
      - filebeat-network
    deploy:
      resources:
        limits:
          cpus: '1.0'
          memory: 1.0G

Redis 部署

Markdown 复制代码

新建一个redis_data目录存储数据
mkdir -p redis_data

YAML 复制代码

# 绑定的IP地址，可以监听本地和外部地址
bind 127.0.0.1 0.0.0.0
# 关闭保护模式，允许外部连接
protected-mode no
# Redis 服务端口
port 6379
# TCP 连接积压队列长度
tcp-backlog 128
# 客户端超时时间，0表示不超时
timeout 0
# TCP 保持活动状态的时间
tcp-keepalive 300
# 不使用守护进程模式
daemonize no
# 不使用系统监督进程
supervised no
# Redis 进程的 PID 文件
#pidfile /var/run/redis_6379.pid
# 日志级别
#loglevel notice
# 日志文件路径
#logfile "/data/redis/logs/redis_master.log"
# 数据库数量
databases 16
# 总是显示 Redis 启动时的标志
always-show-logo yes
# 数据持久化：以下是 RDB 持久化的设置
# 900秒内至少有1个键被修改，进行持久化
save 900 1
# 300秒内至少有10个键被修改，进行持久化
save 300 10
# 60秒内至少有10000个键被修改，进行持久化
save 60 10000
# 如果后台保存出错，停止写操作
stop-writes-on-bgsave-error yes
# 对 RDB 文件进行压缩
rdbcompression yes
# 对 RDB 文件进行校验
rdbchecksum yes
# RDB 文件名
dbfilename dump.rdb
# 从节点可以使用过期数据
slave-serve-stale-data yes
# 从节点只读
slave-read-only no
# 不使用无盘复制
repl-diskless-sync no
# 无盘复制延迟时间
repl-diskless-sync-delay 5
# 启用 TCP 延迟传输，提高性能
repl-disable-tcp-nodelay no
# 从节点优先级，用于故障转移
slave-priority 100
# 连接密码，提高安全性
requirepass oHMU15FrRv!GIPGA
# 最大客户端连接数
maxclients 10000
# 最大内存使用量
maxmemory 6G
# 不使用懒惰淘汰策略
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
# 不启用 AOF 持久化，仅使用 RDB 持久化
appendonly no
# AOF 文件名
appendfilename "appendonly.aof"
# AOF 持久化策略，每秒同步
appendfsync everysec
# 在 AOF 重写时不进行同步
no-appendfsync-on-rewrite no
# AOF 自动重写百分比
auto-aof-rewrite-percentage 100
# AOF 自动重写最小文件大小
auto-aof-rewrite-min-size 64mb
# 加载 AOF 文件时截断不完整的命令
aof-load-truncated yes
# 不使用 RDB 前置 AOF
aof-use-rdb-preamble no
# Lua 脚本执行时间限制
lua-time-limit 5000
# 慢日志阈值
slowlog-log-slower-than 10000
# 慢日志最大长度
slowlog-max-len 128
# 延迟监控阈值
latency-monitor-threshold 0
# 键空间通知事件
notify-keyspace-events ""
# 以下是一些数据结构的优化参数
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
# 开启活跃哈希表重哈希
activerehashing yes
# 客户端输出缓冲区限制
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
# 哈希频率
hz 10
# AOF 重写时增量同步
aof-rewrite-incremental-fsync yes

YAML 复制代码

version: '3.8'

networks:
  redis-network:
    driver: bridge

services:
  redis:
    image: redis:7.0.0
    restart: always
    volumes:
      - ./redis_data:/data
      - ./redis.conf:/etc/redis/redis.conf
    ports:
      - "6379:6379"
    command: redis-server --requirepass "密码"
    networks:
      - redis-network
    deploy:
      resources:
        limits:
          cpus: '1.0'
          memory: 1.5G

ES 创建索引模版、ILM

ES需要提前创建数据源, 数据模版, 参考《 Logstash 指定索引输出至 ES》

Logstash 部署

1、需要下载GeoLite2-City.mmdb

1.1登陆

https://www.maxmind.com/en/account/sign-in

1.2找到 GeoLite City

https://www.maxmind.com/en/accounts/1161290/geoip/downloads

Bash 复制代码

[root@ops /data/ops/elk/logstash]$ pwd
/data/ops/elk/logstash
[root@ops /data/ops/elk/logstash]$ ls
docker-compose.yml  GeoLite2-City.mmdb  logstash.conf  logstash.yml

YAML 复制代码

input {
  redis {
    data_type =>"list"
    key =>"nginx_logs"
    host =>"redisIP"
    port => 6379
    password => "redis的密码"
    db => 0
  }
}

filter {
  if [xff] {
    mutate {
      split => { "[xff]" => "," }  
      add_field => { "client_public_ip" => "%{[xff][0]}" }  # 公网IP存在ipv4、ipv6, 只取第一列
      gsub => [
        "[client_ip]", "\s", ""
      ]
    }
    
    # 确保client_public_ip是字符串类型
    mutate {
      convert => { "client_public_ip" => "string" }
    }
  }

  geoip {
    target => "geoip"
    source => "client_public_ip"
    database => "/usr/share/logstash/GeoLite2-City.mmdb"
    # add_field => [ "[geoip][location]", "%{[geoip][geo][location][lon]}" ]
    # add_field => [ "[geoip][location]", "%{[geoip][geo][location][lat]}" ]
  }

  mutate {
    convert => [ "size", "integer" ]
    convert => [ "status", "integer" ]
    convert => [ "responsetime", "float" ]
    convert => [ "upstreamtime", "float" ]
    convert => [ "[geoip][geo][location][lat]", "float" ]
    convert => [ "[geoip][geo][location][lon]", "float" ]
    add_field => {
        "[geoip][location]" => "%{[geoip][geo][location][lat]},%{[geoip][geo][location][lon]}"
    }
    remove_field => [ "ecs", "agent", "host", "cloud", "@version", "input", "logs_type", "xff" ]
  }

  # 根据http_user_agent来自动处理区分用户客户端系统与版本
  if [http_user_agent] {
    useragent {
      source => "http_user_agent"
      target => "ua"
      remove_field => [ "[ua][minor]","[ua][major]", "[ua][build]", "[ua][patch]", "[ua][os_minor]", "[ua][os_major]", "[geoip][geo][country_iso_code]", "[geoip][geo][continent_code]", "[geoip][geo][region_name]", "[geoip][geo][postal_code]", "[geoip][geo][region_iso_code]" ]
    }
  }
}

# output {
#  stdout { codec => rubydebug }
# }

output {
  elasticsearch {
    hosts => ["ES001IP:9200", "ES002IP:9200", "ES003IP:9200"]
    index => "logstash-nginx-%{+YYYY.MM.dd}"
    template_name => "logstash-nginx"  # 确保模板名称与创建的模板一致
    manage_template => false  # 已手动管理模板
    action => "create"  # 必须使用create操作
  }
}

YAML 复制代码

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "ES001IP:9200", "ES002IP:9200", "ES003IP:9200" ]

YAML 复制代码

version: '3.8'

networks:
  logstash-network:
    driver: bridge

services:
  logstash:
    image: docker.elastic.co/logstash/logstash:8.12.2
    restart: always
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - ./logstash.yml:/usr/share/logstash/config/logstash.yml
      - ./GeoLite2-City.mmdb:/usr/share/logstash/GeoLite2-City.mmdb
    ports:
      - "5044:5044"
    networks:
      - logstash-network
    deploy:
      resources:
        limits:
          cpus: '1.0'
          memory: 2.0G

Grafana 导入并调整模版

你可以直接用我改好的模版:
Grafana ES8.x修改后模版

也可以用 11190的模版自行修改 , 不过需要调整一些字段, 因为8.x 比7.x 多了一个层级

比如geoip.ip 改为 geoip.geo.ip 以下为改造后json, 有些个性化的东西调整了下

在热力图上卡了比较久, 最终配置如下

效果图

告警规则

Markdown 复制代码

[ALERTING] {{$labels.domain}} 域名/URL 监控触发告警

▶ 告警类型：{{$labels.alert_type}}  
▶ 具体 URL：{{$labels.url}}  
▶ 指标描述：{{$labels.message}}  
▶ 触发时间：{{$time | date: "YYYY-MM-DD HH:mm:ss"}}