prometheus整合jmx_exporter 使用jmx_exporter监控Kafka

docker-compose部署kafka集群;单节点单zk-CSDN博客


springboot整合kafka;docker部署kafka-CSDN博客


kafka使用SSL加密和认证--todo_ssl.truststore.location-CSDN博客

复制代码
version: '3.8'

services:
  zookeeper1:
    image: zookeeper:3.9.1
    container_name: zookeeper1-cluster
    hostname: zookeeper-cluster
    ports:
      - "2185:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 10
      ZOOKEEPER_SYNC_LIMIT: 5
    volumes:
      - ./zookeeper-data:/data
      - ./zookeeper-datalog:/datalog
    networks:
      - kafka-cluster-network

  kafka1:
    image: bitnami/kafka:3.8
    container_name: kafka1-cluster
    depends_on:
      - zookeeper1
    ports:
      - "9099:9092" # Kafka 客户端连接端口
      - "9101:9101" # Prometheus 抓取指标的端口
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9099
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
      KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
      KAFKA_CFG_FETCH_MAX_BYTES: 5242880
      KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9101:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
    volumes:
      - ./kafka1-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
    networks:
      - kafka-cluster-network

  kafka2:
    image: bitnami/kafka:3.8
    container_name: kafka2-cluster
    depends_on:
      - zookeeper1
    ports:
      - "9096:9092" # Kafka 客户端连接端口
      - "9102:9102" # Prometheus 抓取指标的端口
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9096
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
      KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
      KAFKA_CFG_FETCH_MAX_BYTES: 5242880
      KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9102:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
    volumes:
      - ./kafka2-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
    networks:
      - kafka-cluster-network

  kafka3:
    image: bitnami/kafka:3.8
    container_name: kafka3-cluster
    depends_on:
      - zookeeper1
    ports:
      - "9097:9092" # Kafka 客户端连接端口
      - "9103:9103" # Prometheus 抓取指标的端口
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-cluster:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.118.20:9097
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_CFG_MESSAGE_MAX_BYTES: 2097152
      KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 3145728
      KAFKA_CFG_FETCH_MAX_BYTES: 5242880
      KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9103:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"
    volumes:
      - ./kafka3-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml
    networks:
      - kafka-cluster-network

  kafka-ui:
    image: provectuslabs/kafka-ui:v0.7.2
    container_name: kafka-ui-cluster
    hostname: kafka-ui-cluster
    ports:
      - "9098:8080"
    depends_on:
      - kafka1
      - kafka2
      - kafka3
    environment:
      KAFKA_CLUSTERS_0_NAME: kafka-cluster
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 192.168.118.20:9096,192.168.118.20:9097,192.168.118.20:9099
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper-cluster:2181
    networks:
      - kafka-cluster-network

networks:
  kafka-cluster-network:
    driver: bridge

下载

jmx_prometheus_javaagent-0.20.0.jar
Central Repository: io/prometheus/jmx/jmx_prometheus_javaagent/0.20.0/

或者https://github.com/prometheus/jmx_exporter/releases/tag/0.20.0


下载 https://github.com/prometheus/jmx_exporter/tree/main/exampleskafka-jmx-exporter-config.yaml

https://github.com/prometheus/jmx_exporter/tree/main/examples
1、在启动kafka时配置代理

2、同时文件也挂载进去

复制代码
KAFKA_OPTS: "-javaagent:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=9102:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml"

    volumes:
      - ./kafka2-data:/bitnami/kafka
      - ./jmx_prometheus_javaagent-0.20.0.jar:/opt/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar
      - ./kafka-jmx-exporter-config.yaml:/opt/jmx_exporter/kafka-jmx-exporter-config.yaml

 - "9103:9103" # Prometheus 抓取指标的端口

启动服务后 检查一下 是否配置成功
http://192.168.118.20:9101/metrics


部署prometheus 和 grafana

复制代码
version: '3.8'

services:
  prometheus:
    image: bitnami/prometheus:3.0.0
    container_name: prometheus
    hostname: prometheus
    ports:
      - "9090:9090" # Prometheus Web UI 端口
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--web.enable-lifecycle'
      - '--storage.tsdb.retention.time=90d'
      - "--storage.tsdb.path=/prometheus"
      - "--web.enable-admin-api"
    restart: always

  grafana:
    image: grafana/grafana:11.3.3
    container_name: grafana
    hostname: grafana
    ports:
      - "3000:3000" # Grafana Web UI 端口
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin # 设置 Grafana 的管理员密码
    volumes:
      - ./grafana-storage:/var/lib/grafana
    restart: always

prometheus.yml  如下配置

global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093


# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  - job_name: 'kafka'
    static_configs:
      - targets:
          - '192.168.118.20:9101' # 对应 kafka1 的 JMX Exporter 端口
          - '192.168.118.20:9102' # 对应 kafka2 的 JMX Exporter 端口
          - '192.168.118.20:9103' # 对应 kafka3 的 JMX Exporter 端口

  - job_name: "node"
    static_configs:
      - targets: ["localhost:9100"]

grafana: 用户名和密码 都是 admin admin

http://192.168.xxx.xxx:3000/


https://grafana.com/grafana/dashboards/11962-kafka-metrics/

https://grafana.com/grafana/dashboards/11962-kafka-metrics/


https://grafana.com/grafana/dashboards/10973-kafka-dashboard333/


Kafka Dashboard | Grafana Labs

该模版需要更新yaml

相关推荐
掘金-我是哪吒12 小时前
分布式微服务系统架构第156集:JavaPlus技术文档平台日更-Java线程池使用指南
java·分布式·微服务·云原生·架构
亲爱的非洲野猪12 小时前
Kafka消息积压的多维度解决方案:超越简单扩容的完整策略
java·分布式·中间件·kafka
活跃家族13 小时前
分布式压测
分布式
前端世界14 小时前
HarmonyOS开发实战:鸿蒙分布式生态构建与多设备协同发布全流程详解
分布式·华为·harmonyos
DavidSoCool15 小时前
RabbitMQ使用topic Exchange实现微服务分组订阅
分布式·微服务·rabbitmq
掘金-我是哪吒16 小时前
分布式微服务系统架构第158集:JavaPlus技术文档平台日更-JVM基础知识
jvm·分布式·微服务·架构·系统架构
东窗西篱梦16 小时前
Redis集群部署指南:高可用与分布式实践
数据库·redis·分布式
Acrel_Fanny16 小时前
Acrel-1000系列分布式光伏监控系统在湖北荆门一马光彩大市场屋顶光伏发电项目中应用
分布式
xufwind17 小时前
spark standlone 集群离线安装
大数据·分布式·spark
半新半旧18 小时前
Redis集群和 zookeeper 实现分布式锁的优势和劣势
redis·分布式·zookeeper