kubernet在prometheus+alertmanager+grafana框架下新增部署loki模块

yunson_Liu2025-03-31 16:33

整个框架拓扑图

Kubernetes Cluster Scrapes Metrics Scrapes Metrics Scrapes Metrics Sends Alerts Scrapes Metrics Queries Metrics Manages Manages Scrapes Metrics Access Dashboards Prometheus Operator Prometheus Instances Alertmanager Instances Grafana kube-state-metrics node-exporter per Node Other Exporters Loki Promtail External Users

1、Loki的安装部署

1）编写有关角色的yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# cat LokiServiceAccount.yaml

yaml 复制代码 
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: loki-promtail-serviceaccount
  namespace: monitoring

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: loki-promtail-clusterrolebinding
  namespace: monitoring
subjects:
    - kind: ServiceAccount
      name: loki-promtail-serviceaccount
      namespace: monitoring
roleRef:
    kind: ClusterRole
    name: loki-promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io

2）创建loki-configmap

yaml 复制代码 
---
apiVersion: v1
data:
  loki.yaml: |-
    auth_enabled: false
    chunk_store_config:
      max_look_back_period: 30d
    compactor:
      shared_store: filesystem
      working_directory: /data/loki/boltdb-shipper-compactor
      compaction_interval: 60m
      retention_enabled: true
      retention_delete_delay: 2h
      retention_delete_worker_count: 150
    ingester:
      chunk_block_size: 262144
      chunk_idle_period: 3m
      chunk_retain_period: 1m
      lifecycler:
        ring:
          kvstore:
            store: inmemory
          replication_factor: 1
      max_transfer_retries: 0
      wal:
        dir: /data/loki/wal          
    limits_config:
      enforce_metric_name: false
      reject_old_samples: true
      reject_old_samples_max_age: 30d
      retention_period: 30d
    schema_config:
      configs:
      - from: "2020-10-24"
        index:
          period: 24h
          prefix: index_
        object_store: filesystem
        schema: v11
        store: boltdb-shipper
    server:
      http_listen_port: 3100
    storage_config:
      boltdb_shipper:
        active_index_directory: /data/loki/boltdb-shipper-active
        cache_location: /data/loki/boltdb-shipper-cache
        cache_ttl: 24h
        shared_store: filesystem
      filesystem:
        directory: /data/loki/chunks
    table_manager:
      retention_deletes_enabled: true
      retention_period: 30d
kind: ConfigMap
metadata:
  name: loki-configmap
  namespace: monitoring
  resourceVersion: '17040827'

3）创建存储类yaml文件

yaml 复制代码 
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    k8s.kuboard.cn/storageType: nfs_client_provisioner
  name: cluster-storage
  resourceVersion: '17031467'
parameters:
  archiveOnDelete: 'false'
provisioner: nfs-cluster-storage
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer

4）创建loki的yaml文件

yaml 复制代码 
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  annotations: {}
  labels:
    app: loki
    k8s.kuboard.cn/addon: system-logs
    k8s.kuboard.cn/name: loki
  name: loki
  namespace: monitoring
  resourceVersion: '17042746'
spec:
  persistentVolumeClaimRetentionPolicy:
    whenDeleted: Retain
    whenScaled: Retain
  podManagementPolicy: OrderedReady
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: loki
      k8s.kuboard.cn/addon: system-logs
      k8s.kuboard.cn/layer: monitor
  serviceName: loki
  template:
    metadata:
      annotations:
        kubectl.kubernetes.io/restartedAt: '2025-03-27T16:07:47+08:00'
        prometheus.io/port: http-metrics
        prometheus.io/scrape: 'true'
      creationTimestamp: null
      labels:
        app: loki
        k8s.kuboard.cn/addon: system-logs
        k8s.kuboard.cn/layer: monitor
    spec:
      containers:
        - args:
            - '-config.file=/etc/loki/loki.yaml'
          image: 'swr.cn-east-2.myhuaweicloud.com/kuboard-dependency/loki:2.7.1'
          imagePullPolicy: IfNotPresent
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /ready
              port: http-metrics
              scheme: HTTP
            initialDelaySeconds: 45
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: loki
          ports:
            - containerPort: 3100
              name: http-metrics
              protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /ready
              port: http-metrics
              scheme: HTTP
            initialDelaySeconds: 45
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources: {}
          securityContext:
            readOnlyRootFilesystem: true
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/loki
              name: config
            - mountPath: /data
              name: storage
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
      serviceAccount: loki-promtail-serviceaccount
      serviceAccountName: loki-promtail-serviceaccount
      terminationGracePeriodSeconds: 4800
      volumes:
        - configMap:
            defaultMode: 420
            name: loki-configmap
          name: config
  updateStrategy:
    type: RollingUpdate
  volumeClaimTemplates:
    - apiVersion: v1
      kind: PersistentVolumeClaim
      metadata:
        annotations:
          k8s.kuboard.cn/pvcType: Dynamic
        creationTimestamp: null
        name: storage
      spec:
        accessModes:
          - ReadWriteMany
        resources:
          requests:
            storage: 5Gi
        storageClassName: cluster-storage
        volumeMode: Filesystem
      status:
        phase: Pending

---
apiVersion: v1
kind: Service
metadata:
  annotations: {}
  labels:
    app: loki
    k8s.kuboard.cn/addon: system-logs
    k8s.kuboard.cn/name: loki
  name: loki
  namespace: monitoring
  resourceVersion: '17029178'
spec:
  clusterIP: 10.68.62.80
  clusterIPs:
    - 10.68.62.80
  internalTrafficPolicy: Cluster
  ipFamilies:
    - IPv4
  ipFamilyPolicy: SingleStack
  ports:
    - name: http-metrics
      port: 3100
      protocol: TCP
      targetPort: http-metrics
  selector:
    app: loki
    k8s.kuboard.cn/addon: system-logs
    k8s.kuboard.cn/layer: monitor
  sessionAffinity: None
  type: ClusterIP

5）启动部署deployment

bash 复制代码 
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f LokiServiceAccount.yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f cluster-storage.yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f loki.yaml

2、Promtail的部署

1）编写serviceaccount文件
root@iZj6c72dzbei17o2cuksmeZ:~/loki# cat promtailServiceAccount.yaml

yaml 复制代码 
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: loki-promtail-clusterrole
  namespace: monitoring
rules:
  - apiGroups: [""]
    resources:
    - nodes
    - services
    - pods
    verbs:
    - get
    - watch
    - list
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: loki-promtail-serviceaccount
  namespace: monitoring

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: loki-promtail-clusterrolebinding
  namespace: monitoring 
subjects:
    - kind: ServiceAccount
      name: loki-promtail-serviceaccount
      namespace: monitoring 
roleRef:
    kind: ClusterRole
    name: loki-promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io```

2）编写promtail-configmap文件

```yaml
---
apiVersion: v1
data:
  promtail.yaml: |-
    client:
      backoff_config:
        max_period: 5s
        max_retries: 20
        min_period: 100ms
      batchsize: 102400
      batchwait: 1s
      external_labels: {}
      timeout: 10s
    positions:
      filename: /run/promtail/positions.yaml
    server:
      http_listen_port: 3101
    target_config:
      sync_period: 10s
    scrape_configs:
    - job_name: kubernetes-pods-name
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels:
        - __meta_kubernetes_pod_label_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-app
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        source_labels:
        - __meta_kubernetes_pod_label_name
      - source_labels:
        - __meta_kubernetes_pod_label_app
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-direct-controllers
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: drop
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-indirect-controller
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: keep
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - action: replace
        regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-static
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: ''
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_label_component
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
        - __meta_kubernetes_pod_container_name
        target_label: __path__
kind: ConfigMap
metadata:
  name: promtail-configmap
  namespace: monitoring
  resourceVersion: '17042330'

3）启动deployment

bash 复制代码 
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f promtailServiceAccount.yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f promtail.yaml

3、web界面操作

1）登录web界面，导入loki数据源

点击Data sources --> +Add new data source

选择loki项

自定义相关的参数项

滑倒最下面点击保存

2）自定义条件查看日志信息

3）导入Dashboards

选中Dashboards-->New-->import

输入模块id-->加载

自定义Dashboard名称--id--选择loki数据源

最后点击导入

相关推荐
计算机毕设定制辅导-无忧学长12 小时前
Grafana 与 InfluxDB 可视化深度集成（二）
信息可视化·数据分析·grafana
云游17 小时前
大模型性能指标的监控系统（prometheus3.5.0）和可视化工具(grafana12.1.0)基础篇
grafana·prometheus·可视化·监控
qq_232045572 天前
非容器方式安装Prometheus和Grafana，以及nginx配置访问Grafana
nginx·grafana·prometheus
测试开发Kevin2 天前
详解grafana k6 中stage的核心概念与作用
测试工具·压力测试·grafana
夜莺云原生监控2 天前
Prometheus 监控 Kubernetes Cluster 最新极简教程
容器·kubernetes·prometheus
SRETalk3 天前
Prometheus 监控 Kubernetes Cluster 最新极简教程
kubernetes·prometheus
川石课堂软件测试3 天前
JMeter并发测试与多进程测试
功能测试·jmeter·docker·容器·kubernetes·单元测试·prometheus
SRETalk3 天前
夜莺监控的几种架构模式详解
prometheus·victoriametrics·nightingale·夜莺监控
天翼云开发者社区4 天前
Grafana无法启动修复解决
grafana
Ditglu.4 天前
使用Prometheus + Grafana + node_exporter实现Linux服务器性能监控
服务器·grafana·prometheus
热门推荐
01UV安装并设置国内源02KGG转MP3工具|非KGM文件|解密音频03【2025.08.06最新版】Android Studio下载、安装及配置记录（自动下载sdk）04Qwen3-Coder 快速上手教程 | Qwen Code + Claude Code05蜘蛛磁力 搜索引擎大全，如何使用蜘蛛磁力查找磁力链接062025最新国内服务器可用docker源仓库地址大全（2025年8月更新）07TRAE 规则（Rules）配置指南：个人习惯、团队规范与最佳实践08NVIDIA显卡驱动、CUDA、cuDNN 和 TensorRT 版本匹配指南09全球最强模型Grok4，国内已可免费使用！（附教程）10TRAE Rules 实践：为项目配置 6A 工作流