helm 部署 Kube-Prometheus + Grafana + 钉钉告警部署 Kube-Prometheus

背景

角色 IP K8S 版本 容器运行时
k8s-master-1 172.16.16.108 v1.24.1 containerd://1.6.8
k8s-node-1 172.16.16.109 v1.24.1 containerd://1.6.8
k8s-node-2 172.16.16.110 v1.24.1 containerd://1.6.8

安装 kube-prometheus

mkdir -p /data/yaml/kube-prometheus/prometheus && cd /data/yaml/kube-prometheus/prometheus

# 添加 bitnami charts 仓库
helm repo add bitnami https://charts.bitnami.com/bitnami

helm search repo kube-prometheus

helm pull bitnami/kube-prometheus --version 8.3.0

tar -zxvf kube-prometheus-8.3.0.tgz

cat > my-values.yaml << EOF
global:
  storageClass: "nfs-client"  # 默认 storageClass

prometheus:
  service:
    type: NodePort      # 配置 NodePort
    nodePorts: 
      http: 30090       # 配置 NodePort 端口
  persistence:
    enabled: true       # 开启持久化
    size: 9Gi           # 存储大小

alertmanager:
  service:
    type: NodePort      # 配置 NodePort
    nodePorts: 
      http: 30093       # 配置 NodePort 端口
  persistence:
    enabled: true       # 开启持久化
    size: 9Gi           # 存储大小
  config:
    route:
      receiver: 'devops'   # 告警接收者
      routes:
        - match:
          receiver: 'devops'
    receivers:
      - name: 'devops'       # 告警接收者
        webhook_configs:
        - url: 'http://prometheus-webhook-dingtalk.kube-prometheus:8060/dingtalk/devops/send'     # 注意这里的 devops 需要与 prometheus-webhook-dingtalk 中的 --ding.profile 值相同
          send_resolved: true
EOF

# 创建命名空间
kubectl create ns kube-prometheus

# 测试
helm install --namespace kube-prometheus prometheus -f my-values.yaml --dry-run  kube-prometheus

# 启动
helm install --namespace kube-prometheus prometheus -f my-values.yaml  kube-prometheus

# 查看
helm -n kube-prometheus ls

kubectl -n kube-prometheus get pod

访问 Prometheus

http://172.16.16.108:30090/

配置 Pod 告警策略

mkdir -p /data/yaml/kube-prometheus/prometheus/rules && cd /data/yaml/kube-prometheus/prometheus/rules

cat >> k8s-pod-rules.yaml << -'EOF'
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    prometheus-name: kube-prometheus-prometheus
    managed-by: prometheus-operator          
  name: prometheus-k8s-pod-rules
  namespace:  kube-prometheus
spec:
  groups:
  - name: PodMemUsage
    rules:
    - alert: Pod内存使用率告警
      expr: sum by (pod, namespace, job, container) (container_memory_working_set_bytes{pod!="",container !=""}) / sum by (pod, namespace, job, container) (container_spec_memory_limit_bytes{pod!="",container !=""}) * 100 != +Inf > 95
      for: 1m
      labels:
        severity: 紧急告警
        service: pods
      annotations:
        description: "{{$labels.instance}}: 当前Pod内存使用率大于95% ,使用率为: {{ $value }}"
        summary: "Pod:{{ $labels.pod }} 检测到内存使用率超过limit值95%"  

  - name: Pod_cpu
    rules:
    - alert: Pod_CPU使用率告警
      expr: sum(irate(container_cpu_usage_seconds_total{pod!="",container !=""}[1m])) by (container, pod) / (sum(container_spec_cpu_quota{pod!="",container !=""}/100000) by (container, pod)) * 100 > 130
      for: 1m
      labels:
        severity: 严重告警
        service: pods
      annotations:
        description: "{{$labels.pod}}: 一分钟内Pod的cpu使用率大于130%,当前的使用率为: {{ $value }}"  


  - name: Pod_Network_rx
    rules:
    - alert: Pod网络IO(rx方向)告警
      expr: (sum (rate (container_network_receive_bytes_total{pod!=""}[1m])) by (pod)) / 1024  / 1024 > 200
      for: 1m
      labels:
        severity: 严重告警
        service: pods
      annotations:
        description: "{{$labels.instance}}: 一分钟内Pod的Pod网络IO(rx方向)大于200Mbps,当前的值为: {{ $value }} Mbps"
        summary: "Pod:{{ $labels.pod }} 检测到一分钟内网络IO(rx方向)过高"  

  - name: Pod_Network_tx
    rules:
    - alert: Pod网络IO(tx方向)告警
      expr: (sum (rate (container_network_transmit_bytes_total{pod!=""}[1m])) by (pod)) / 1024 / 1024 > 200
      for: 1m
      labels:
        severity: 严重告警
        service: pods
      annotations:
        description: "{{$labels.instance}}: 一分钟内Pod的Pod网络IO(tx方向)大于200Mbps,当前的值为: {{ $value }} Mbps"
        summary: "检测到一分钟内Pod网络IO(tx方向)过高"  

  - name: imagepullbackoff
    rules:
    - alert: 拉取镜像失败
      expr: kube_pod_container_status_waiting_reason{reason="ImagePullBackOff"} == 1
      for: 1m
      labels:
        severity: 紧急告警
      annotations:
        summary: "POD:{{ $labels.pod }} 拉取镜像失败,无法创建容器"
        description: "请确认镜像是否存在"
        
  - name: Pod_Start_Exception
    rules:
    - alert: POD 资源配置不正确
      expr: sum by (namespace, pod) (kube_pod_status_phase{ phase=~"Pending|Unknown"}) == 1
      for: 15s
      labels:
        severity: 紧急告警
      annotations:
        summary: "POD:{{ $labels.pod }} 启动失败,请及时查看"
        description: "POD 无法正常启动,请查看资源是否配置正确"

  - name: crashloopbackoff
    rules:
    - alert: POD启动失败
      expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} == 1
      for: 1m
      labels:
        severity: 紧急告警
      annotations:
        summary: "POD:{{ $labels.pod }} 启动失败,请查看程序日志"
        description: "确认配置参数是否正确" 
-EOF

kubectl apply -f k8s-pod-rules.yaml

# 检查
kubectl -n kube-prometheus get cm
相关推荐
川石课堂软件测试18 小时前
性能测试|docker容器下搭建JMeter+Grafana+Influxdb监控可视化平台
运维·javascript·深度学习·jmeter·docker·容器·grafana
逻辑与&&1 天前
[Prometheus学习笔记]从架构到案例,一站式教程
笔记·学习·prometheus
Walden-20202 天前
构建基于 DCGM-Exporter, Node exporter,PROMETHEUS 和 GRAFANA 构建算力监控系统
docker·容器·grafana·prometheus
牛角上的男孩3 天前
部署Prometheus、Grafana、Zipkin、Kiali监控度量Istio
grafana·prometheus·istio
福大大架构师每日一题4 天前
文心一言 VS 讯飞星火 VS chatgpt (383)-- 算法导论24.5 3题
prometheus
小安运维日记6 天前
Linux云计算 |【第五阶段】CLOUD-DAY10
linux·运维·云计算·k8s·grafana·prometheus
川石课堂软件测试7 天前
性能测试|linux服务器搭建JMeter+Grafana+Influxdb监控可视化平台
linux·服务器·python·jmeter·机器学习·grafana·1024程序员节
福大大架构师每日一题7 天前
29.2 golang实战项目log2metrics架构说明
架构·prometheus
花开了¥7 天前
prometheus 快速入门
prometheus
無爲謂7 天前
入门 | Kafka数据使用vector消费到Loki中使用grafana展示
分布式·kafka·vector·grafana·loki·日志存储·es限制解决办法