K8S学习之基础三十七：prometheus监控node资源

云上艺旅2025-03-20 17:33

Prometheus v2.2.1

​ 编写yaml文件，包含创建ns、configmap、deployment、service

复制代码 
# 创建monitoring空间
vi prometheus-ns.yaml 
apiVersion: v1
kind: Namespace
metadata:
  name: monitor-sa

# 创建SA并绑定权限
kubectl create serviceaccount monitor -n monitor-sa
kubectl create clusterrolebinding monitor-clusterrolebinding -n monitor-sa --clusterrole=cluster-admin  --serviceaccount=monitor-sa:monitor
kubectl create clusterrolebinding monitor-clusterrolebinding-1  -n monitor-sa --clusterrole=cluster-admin   --user=system:serviceaccount:monitor-sa:monitor

# 创建cm、deployment、svc
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    app: prometheus
  name: prometheus-config
  namespace: monitor-sa
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      scrape_timeout: 10s
      evaluation_interval: 1m
    scrape_configs:
    - job_name: 'kubernetes-node'
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
        action: replace
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
    - job_name: 'kubernetes-node-cadvisor'
      kubernetes_sd_configs:
      - role:  node
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
    - job_name: 'kubernetes-apiserver'
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_po
rt_name]
        action: keep
        regex: default;kubernetes;https
    - job_name: 'kubernetes-service-endpoints'
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name 
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-server
  namespace: monitoring
  labels:
    app: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
      component: server
    #matchExpressions:
    #- {key: app, operator: In, values: [prometheus]}
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: monitor-sa
  labels:
    app: prometheus
  annotations:        # 增加注解，可被prometheus监控到
    prometheus.io/scrape: "true"    
spec:
  type: NodePort
  ports:
    - port: 9090
      targetPort: 9090
      protocol: TCP
  selector:
    app: prometheus
    component: server
    
[root@mast01 prometheus]# kubectl get pod -n monitor-sa
NAME                         READY   STATUS    RESTARTS   AGE
prometheus-dfdfb9d79-h4tk4   1/1     Running   0          22h
[root@mast01 prometheus]# kubectl get svc -n monitor-sa
NAME         TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
prometheus   NodePort   10.109.44.37   <none>        9090:30090/TCP   22h
[root@mast01 prometheus]# curl 10.109.44.37:9090
<a href="/graph">Found</a>.

[root@mast01 prometheus]# kubectl get ep -n monitor-sa
NAME         ENDPOINTS            AGE
prometheus   10.244.140.67:9090   22h
[root@mast01 prometheus]# curl 172.16.80.131:30090
<a href="/graph">Found</a>.

浏览器访问 172.16.80.131:30090

node-exporter组件安装和配置

​ node-exporter可以采集机器（物理机、虚拟机、云主机等）的监控指标数据，能够采集到的指标包括CPU, 内存，磁盘，网络，文件数等信息。

复制代码 
# 上传node-exporter.tar.gz到harbor
docker load -i node-exporter.tar.gz
docker tag prom/node-exporter:v0.16.0 172.16.80.140/node-exporter/node-exporter:v0.16.0
docker push 172.16.80.140/node-exporter/node-exporter:v0.16.0
# 创建daemon控制器的yaml
[root@mast01 prometheus]#  vi node-export.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: monitor-sa
  labels:
    name: node-exporter
spec:
  selector:
    matchLabels:
     name: node-exporter
  template:
    metadata:
      labels:
        name: node-exporter
    spec:
      hostPID: true
      hostIPC: true
      hostNetwork: true
      containers:
      - name: node-exporter
        image: 172.16.80.140/node-exporter/node-exporter:v0.16.0
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9100
        resources:
          requests:
            cpu: 0.15
        securityContext:
          privileged: true
        args:
        - --path.procfs
        - /host/proc
        - --path.sysfs
        - /host/sys
        - --collector.filesystem.ignored-mount-points
        - '"^/(sys|proc|dev|host|etc)($|/)"'
        volumeMounts:
        - name: dev
          mountPath: /host/dev
        - name: proc
          mountPath: /host/proc
        - name: sys
          mountPath: /host/sys
        - name: rootfs
          mountPath: /rootfs
      tolerations:
      - key: "node-role.kubernetes.io/control-plane"
        operator: "Exists"
        effect: "NoSchedule"
      volumes:
        - name: proc
          hostPath:
            path: /proc
        - name: dev
          hostPath:
            path: /dev
        - name: sys
          hostPath:
            path: /sys
        - name: rootfs
          hostPath:
            path: /
# hostNetwork、hostIPC、hostPID都为True时，表示这个Pod里的所有容器，会直接使用宿主机的网络，直接与宿主机进行IPC（进程间通信）通信，可以看到宿主机里正在运行的所有进程。加入了hostNetwork:true会直接将我们的宿主机的9100端口映射出来，从而不需要创建service 在我们的宿主机上就会有一个9100的端口

​ 该yaml会在每个节点上部署一个node-exporter

复制代码 
[root@mast01 prometheus]# netstat -an | grep 9100
tcp6       0      0 :::9100                 :::*                    LISTEN
[root@mast01 prometheus]# kubectl get pods -n monitor-sa -owide
NAME                 READY STATUS  RESTARTS AGE IP            NODE   NOMINATED NODE   READINESS GATES
node-exporter-5ms5j  1/1   Running 0        7s  172.16.80.133 node02 <none>    <none>
node-exporter-dgg2z  1/1   Running 0        9s  172.16.80.131 mast01 <none>    <none>
node-exporter-k4mf5  1/1   Running 0        6s  172.16.80.132 node01 <none>    <none>

​ 通过curl可获取主机信息

复制代码 
[root@mast01 prometheus]# curl http://172.16.80.131:9100/metrics | more
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 0.00010856
go_gc_duration_seconds{quantile="0.25"} 0.00010856
go_gc_duration_seconds{quantile="0.5"} 0.000879866
go_gc_duration_seconds{quantile="0.75"} 0.000879866
go_gc_duration_seconds{quantile="1"} 0.000879866
go_gc_duration_seconds_sum 0.000988426
go_gc_duration_seconds_count 2
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 6
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.9.6"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 1.863448e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 5.928912e+06
......
curl http://172.16.80.131:9100/metrics | grep node_cpu_seconds   # 显示cpu使用情况
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 91593  100 91593    0     0  6934k      0 --:--:-- # HELP node_cpu_seconds_total Seconds the cpus spent in each mode.
--:--# TYPE node_cpu_seconds_total counter     # counter类型，只增不减
:--node_cpu_seconds_total{cpu="0",mode="idle"} 74516.69
 -node_cpu_seconds_total{cpu="0",mode="iowait"} 13.52
-:node_cpu_seconds_total{cpu="0",mode="irq"} 0
--node_cpu_seconds_total{cpu="0",mode="nice"} 0.03
:-node_cpu_seconds_total{cpu="0",mode="softirq"} 76.96
- node_cpu_seconds_total{cpu="0",mode="steal"} 0
74node_cpu_seconds_total{cpu="0",mode="system"} 601.74
53node_cpu_seconds_total{cpu="0",mode="user"} 1234.91
knode_cpu_seconds_total{cpu="1",mode="idle"} 74511.08

node_cpu_seconds_total{cpu="1",mode="iowait"} 24.89
node_cpu_seconds_total{cpu="1",mode="irq"} 0
node_cpu_seconds_total{cpu="1",mode="nice"} 0.18
node_cpu_seconds_total{cpu="1",mode="softirq"} 86.84
node_cpu_seconds_total{cpu="1",mode="steal"} 0
node_cpu_seconds_total{cpu="1",mode="system"} 598.25
node_cpu_seconds_total{cpu="1",mode="user"} 1217.33

curl http://172.16.80.131:9100/metrics | grep node_load # 最近一分钟内负载使用情况
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0# HELP node_load1 1m load average.
# TYPE node_load1 gauge   # gauge类型，可增可减
node_load1 0.27
# HELP node_load15 15m load average.
# TYPE node_load15 gauge
node_load15 0.42
# HELP node_load5 5m load average.
# TYPE node_load5 gauge
node_load5 0.43
100 91584  100 91584    0     0  8262k      0 --:--:-- --:--:-- --:--:-- 8943k

通过prometheus，可以看到监控对象

Graph选择对应指标和周期，可以打印出指标图形

prometheus热加载：

svc配置修改后，prometheus并不会马上更新配置，可以通过reload这个svc对应pod的配置，这样比较安全快速的使prometheus进行了热加载，并不会对其他产生影响

复制代码 
# 获取svc对应的pod的ip
kubectl get pods -owide
# 热加载
curl -X POST http://podip:9090/-/reload

如果直接删除prometheus，加载配置，也可以达到更新目的，但这种暴力加载会使监控数据丢失，不建议使用

相关推荐
lx学习17 分钟前
嵌入式c学习六
c语言·数据结构·学习
myzzb3 小时前
python字符级差异分析并生成 Word 报告 自然语言处理断句
python·学习·自然语言处理·word·snownlp
土豆土豆，我是洋芋5 小时前
《大语言模型》学习笔记（二）
笔记·学习·语言模型·datawhale
猪猪成5 小时前
【图论】FLOYD弗洛伊德算法-最短路径
学习·算法·图论
小程同学>o<5 小时前
嵌入式开发之STM32学习笔记day08
笔记·stm32·学习
虾球xz6 小时前
游戏引擎学习第170天
javascript·学习·游戏引擎
虾球xz6 小时前
游戏引擎学习第168天
前端·学习·游戏引擎
虾球xz6 小时前
游戏引擎学习第167天
学习·游戏引擎
Akamai中国7 小时前
应用商店上新：Couchbase Enterprise Server集群
运维·服务器·sql·云原生·云计算·云服务
桂月二二7 小时前
云原生服务网格：微服务通讯的量子纠缠革命
微服务·云原生·架构
热门推荐
013.18练习02从零安装 LLaMA-Factory 微调 Qwen 大模型成功及所有的坑03DeepSeek各版本说明与优缺点分析04LLM主流框架：Causal Decoder、Prefix Decoder和Encoder-Decoder05如何在WPS和Word/Excel中直接使用DeepSeek功能06本地部署DeepSeek教程（Mac版本）07实现鼠标键盘动作录制与重复播放的工具08KGG转MP3工具|非KGM文件|解密音频09本地化部署AI知识库：基于Ollama+DeepSeek+AnythingLLM保姆级教程10nginx性能优化有哪些方式？