prometheus整合jmx_exporter 使用jmx_exporter监控Kafka

敏君宝爸2025-04-11 8:57

docker-compose部署kafka集群；单节点单zk-CSDN博客

springboot整合kafka；docker部署kafka-CSDN博客

kafka使用SSL加密和认证--todo_ssl.truststore.location-CSDN博客

复制代码 
version: '3.8'

services:
  zookeeper1:
    image: zookeeper:3.9.1
    container_name: zookeeper1-cluster
    hostname: zookeeper-cluster
    ports:
      - "2185:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 10
      ZOOKEEPER_SYNC_LIMIT: 5
    volumes:
      - ./zookeeper-data:/data
      - ./zookeeper-datalog:/datalog
    networks:
      - kafka-cluster-network

  kafka1:
    image: bitnami/kafka:3.8
    container_name: kafka1-cluster
    depends_on:
      - zookeeper1
    ports:
      - "9099:9092" # Kafka 客户端连接端口
      - "9101:9101" # Prometheus 抓取指标的端口
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9099
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
      KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
      KAFKA_CFG_FETCH_MAX_BYTES: 5242880
      KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9101:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
    volumes:
      - ./kafka1-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
    networks:
      - kafka-cluster-network

  kafka2:
    image: bitnami/kafka:3.8
    container_name: kafka2-cluster
    depends_on:
      - zookeeper1
    ports:
      - "9096:9092" # Kafka 客户端连接端口
      - "9102:9102" # Prometheus 抓取指标的端口
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9096
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
      KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
      KAFKA_CFG_FETCH_MAX_BYTES: 5242880
      KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9102:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
    volumes:
      - ./kafka2-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
    networks:
      - kafka-cluster-network

  kafka3:
    image: bitnami/kafka:3.8
    container_name: kafka3-cluster
    depends_on:
      - zookeeper1
    ports:
      - "9097:9092" # Kafka 客户端连接端口
      - "9103:9103" # Prometheus 抓取指标的端口
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9097
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
      KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
      KAFKA_CFG_FETCH_MAX_BYTES: 5242880
      KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9103:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
    volumes:
      - ./kafka3-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
    networks:
      - kafka-cluster-network

  kafka-ui:
    image: provectuslabs/kafka-ui:v0.7.2
    container_name: kafka-ui-cluster
    hostname: kafka-ui-cluster
    ports:
      - "9098:8080"
    depends_on:
      - kafka1
      - kafka2
      - kafka3
    environment:
      KAFKA_CLUSTERS_0_NAME: kafka-cluster
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 192.168.118.20:9096,192.168.118.20:9097,192.168.118.20:9099
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper-cluster:2181
    networks:
      - kafka-cluster-network

networks:
  kafka-cluster-network:
    driver: bridge

下载

jmx_prometheus_javaagent-0.20.0.jar
Central Repository: io/prometheus/jmx/jmx_prometheus_javaagent/0.20.0/

或者https://github.com/prometheus/jmx_exporter/releases/tag/0.20.0


下载 https://github.com/prometheus/jmx_exporter/tree/main/exampleskafka-jmx-exporter-config.yaml

https://github.com/prometheus/jmx_exporter/tree/main/examples
1、在启动kafka时配置代理

2、同时文件也挂载进去

复制代码 
KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9102:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"

    volumes:
      - ./kafka2-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml

 - "9103:9103" # Prometheus 抓取指标的端口

启动服务后 检查一下 是否配置成功
http://192.168.118.20:9101/metrics

部署prometheus 和 grafana

复制代码 
version: '3.8'

services:
  prometheus:
    image: bitnami/prometheus:3.0.0
    container_name: prometheus
    hostname: prometheus
    ports:
      - "9090:9090" # Prometheus Web UI 端口
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--web.enable-lifecycle'
      - '--storage.tsdb.retention.time=90d'
      - "--storage.tsdb.path=/prometheus"
      - "--web.enable-admin-api"
    restart: always

  grafana:
    image: grafana/grafana:11.3.3
    container_name: grafana
    hostname: grafana
    ports:
      - "3000:3000" # Grafana Web UI 端口
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin # 设置 Grafana 的管理员密码
    volumes:
      - ./grafana-storage:/var/lib/grafana
    restart: always

prometheus.yml  如下配置

global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093


# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  - job_name: 'kafka'
    static_configs:
      - targets:
          - '192.168.118.20:9101' # 对应 kafka1 的 JMX Exporter 端口
          - '192.168.118.20:9102' # 对应 kafka2 的 JMX Exporter 端口
          - '192.168.118.20:9103' # 对应 kafka3 的 JMX Exporter 端口

  - job_name: "node"
    static_configs:
      - targets: ["localhost:9100"]

grafana: 用户名和密码 都是 admin admin

http://192.168.xxx.xxx:3000/

https://grafana.com/grafana/dashboards/11962-kafka-metrics/

https://grafana.com/grafana/dashboards/11962-kafka-metrics/

https://grafana.com/grafana/dashboards/10973-kafka-dashboard333/

Kafka Dashboard | Grafana Labs

该模版需要更新yaml

相关推荐
lbb 小魔仙19 小时前
OpenClaw + cpolar 实战：远程 NAS、分享小游戏、RDP，再配置公网 AI 入口
数据库·人工智能·redis·oracle·prometheus
Thomas.Sir19 小时前
第21课：PyTorch｜GPU多卡训练与分布式训练基础【让多卡并行成为你的加速引擎】
人工智能·pytorch·分布式
Cicada12820 小时前
库存消息消费的正确性设计——从幂等窗口到批量流水线
分布式·系统架构
Nano叶落21 小时前
Kafka 消费积压排查入门：Docker 搭环境亲手制造一次 ‘消息堵死‘，10 分钟看懂 Lag
kafka
zhoupenghui1681 天前
Golang pprof 工具详解：监控、压测与调优实战指南
prometheus·pprof
Francek Chen2 天前
【大数据处理与分析】数据仓库Hive：04 数据仓库Hive概述
大数据·数据仓库·hive·hadoop·分布式
Gl�ria2 天前
Hadoop/YARN 集群缩容：下线DN节点
大数据·hadoop·分布式
吉甫作诵2 天前
Kafka 集群安装与运维实战：消费组排查、Offset 重置与副本重分配
大数据·运维·分布式·kafka·消息队列
imDwAaY2 天前
消息队列四大核心问题：顺序性、幂等性、可靠性与一致性
学习·kafka·rabbitmq
热门推荐
01国内可直接用、免费额度/永久免费的大模型API清单（含 SiliconFlow、火山、阿里、智谱、百度、Kimi、DeepSeek、DMXAPI 等）02GitHub 镜像站点03历年考研数学一、数学二、数学三真题试卷及答案PDF04蜘蛛磁力 搜索引擎大全，如何使用蜘蛛磁力查找磁力链接05微信历史版本含下载地址（ Windows PC | 安卓 | MAC ）及设置微信不更新06Telegram APK 下载安装完整指南 — 2026年最新072026 国产 AI 大模型横评：DeepSeek、通义千问、Kimi、文心一言、星火、豆包谁更能打？082026 爱知・名古屋亚运会怎么看？直播平台+赛程安排分享09人工智能（AI）发展史：详细里程碑10ZCode 周末送额度活动开启：3 亿 Token 免费领取