kubernet在prometheus+alertmanager+grafana框架下新增部署loki模块

yunson_Liu2025-03-31 16:33

整个框架拓扑图

Kubernetes Cluster Scrapes Metrics Scrapes Metrics Scrapes Metrics Sends Alerts Scrapes Metrics Queries Metrics Manages Manages Scrapes Metrics Access Dashboards Prometheus Operator Prometheus Instances Alertmanager Instances Grafana kube-state-metrics node-exporter per Node Other Exporters Loki Promtail External Users

1、Loki的安装部署

1）编写有关角色的yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# cat LokiServiceAccount.yaml

yaml 复制代码 
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: loki-promtail-serviceaccount
  namespace: monitoring

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: loki-promtail-clusterrolebinding
  namespace: monitoring
subjects:
    - kind: ServiceAccount
      name: loki-promtail-serviceaccount
      namespace: monitoring
roleRef:
    kind: ClusterRole
    name: loki-promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io

2）创建loki-configmap

yaml 复制代码 
---
apiVersion: v1
data:
  loki.yaml: |-
    auth_enabled: false
    chunk_store_config:
      max_look_back_period: 30d
    compactor:
      shared_store: filesystem
      working_directory: /data/loki/boltdb-shipper-compactor
      compaction_interval: 60m
      retention_enabled: true
      retention_delete_delay: 2h
      retention_delete_worker_count: 150
    ingester:
      chunk_block_size: 262144
      chunk_idle_period: 3m
      chunk_retain_period: 1m
      lifecycler:
        ring:
          kvstore:
            store: inmemory
          replication_factor: 1
      max_transfer_retries: 0
      wal:
        dir: /data/loki/wal          
    limits_config:
      enforce_metric_name: false
      reject_old_samples: true
      reject_old_samples_max_age: 30d
      retention_period: 30d
    schema_config:
      configs:
      - from: "2020-10-24"
        index:
          period: 24h
          prefix: index_
        object_store: filesystem
        schema: v11
        store: boltdb-shipper
    server:
      http_listen_port: 3100
    storage_config:
      boltdb_shipper:
        active_index_directory: /data/loki/boltdb-shipper-active
        cache_location: /data/loki/boltdb-shipper-cache
        cache_ttl: 24h
        shared_store: filesystem
      filesystem:
        directory: /data/loki/chunks
    table_manager:
      retention_deletes_enabled: true
      retention_period: 30d
kind: ConfigMap
metadata:
  name: loki-configmap
  namespace: monitoring
  resourceVersion: '17040827'

3）创建存储类yaml文件

yaml 复制代码 
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    k8s.kuboard.cn/storageType: nfs_client_provisioner
  name: cluster-storage
  resourceVersion: '17031467'
parameters:
  archiveOnDelete: 'false'
provisioner: nfs-cluster-storage
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer

4）创建loki的yaml文件

yaml 复制代码 
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  annotations: {}
  labels:
    app: loki
    k8s.kuboard.cn/addon: system-logs
    k8s.kuboard.cn/name: loki
  name: loki
  namespace: monitoring
  resourceVersion: '17042746'
spec:
  persistentVolumeClaimRetentionPolicy:
    whenDeleted: Retain
    whenScaled: Retain
  podManagementPolicy: OrderedReady
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: loki
      k8s.kuboard.cn/addon: system-logs
      k8s.kuboard.cn/layer: monitor
  serviceName: loki
  template:
    metadata:
      annotations:
        kubectl.kubernetes.io/restartedAt: '2025-03-27T16:07:47+08:00'
        prometheus.io/port: http-metrics
        prometheus.io/scrape: 'true'
      creationTimestamp: null
      labels:
        app: loki
        k8s.kuboard.cn/addon: system-logs
        k8s.kuboard.cn/layer: monitor
    spec:
      containers:
        - args:
            - '-config.file=/etc/loki/loki.yaml'
          image: 'swr.cn-east-2.myhuaweicloud.com/kuboard-dependency/loki:2.7.1'
          imagePullPolicy: IfNotPresent
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /ready
              port: http-metrics
              scheme: HTTP
            initialDelaySeconds: 45
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: loki
          ports:
            - containerPort: 3100
              name: http-metrics
              protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /ready
              port: http-metrics
              scheme: HTTP
            initialDelaySeconds: 45
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources: {}
          securityContext:
            readOnlyRootFilesystem: true
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/loki
              name: config
            - mountPath: /data
              name: storage
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
      serviceAccount: loki-promtail-serviceaccount
      serviceAccountName: loki-promtail-serviceaccount
      terminationGracePeriodSeconds: 4800
      volumes:
        - configMap:
            defaultMode: 420
            name: loki-configmap
          name: config
  updateStrategy:
    type: RollingUpdate
  volumeClaimTemplates:
    - apiVersion: v1
      kind: PersistentVolumeClaim
      metadata:
        annotations:
          k8s.kuboard.cn/pvcType: Dynamic
        creationTimestamp: null
        name: storage
      spec:
        accessModes:
          - ReadWriteMany
        resources:
          requests:
            storage: 5Gi
        storageClassName: cluster-storage
        volumeMode: Filesystem
      status:
        phase: Pending

---
apiVersion: v1
kind: Service
metadata:
  annotations: {}
  labels:
    app: loki
    k8s.kuboard.cn/addon: system-logs
    k8s.kuboard.cn/name: loki
  name: loki
  namespace: monitoring
  resourceVersion: '17029178'
spec:
  clusterIP: 10.68.62.80
  clusterIPs:
    - 10.68.62.80
  internalTrafficPolicy: Cluster
  ipFamilies:
    - IPv4
  ipFamilyPolicy: SingleStack
  ports:
    - name: http-metrics
      port: 3100
      protocol: TCP
      targetPort: http-metrics
  selector:
    app: loki
    k8s.kuboard.cn/addon: system-logs
    k8s.kuboard.cn/layer: monitor
  sessionAffinity: None
  type: ClusterIP

5）启动部署deployment

bash 复制代码 
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f LokiServiceAccount.yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f cluster-storage.yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f loki.yaml

2、Promtail的部署

1）编写serviceaccount文件
root@iZj6c72dzbei17o2cuksmeZ:~/loki# cat promtailServiceAccount.yaml

yaml 复制代码 
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: loki-promtail-clusterrole
  namespace: monitoring
rules:
  - apiGroups: [""]
    resources:
    - nodes
    - services
    - pods
    verbs:
    - get
    - watch
    - list
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: loki-promtail-serviceaccount
  namespace: monitoring

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: loki-promtail-clusterrolebinding
  namespace: monitoring 
subjects:
    - kind: ServiceAccount
      name: loki-promtail-serviceaccount
      namespace: monitoring 
roleRef:
    kind: ClusterRole
    name: loki-promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io```

2）编写promtail-configmap文件

```yaml
---
apiVersion: v1
data:
  promtail.yaml: |-
    client:
      backoff_config:
        max_period: 5s
        max_retries: 20
        min_period: 100ms
      batchsize: 102400
      batchwait: 1s
      external_labels: {}
      timeout: 10s
    positions:
      filename: /run/promtail/positions.yaml
    server:
      http_listen_port: 3101
    target_config:
      sync_period: 10s
    scrape_configs:
    - job_name: kubernetes-pods-name
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels:
        - __meta_kubernetes_pod_label_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-app
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        source_labels:
        - __meta_kubernetes_pod_label_name
      - source_labels:
        - __meta_kubernetes_pod_label_app
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-direct-controllers
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: drop
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-indirect-controller
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: keep
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - action: replace
        regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-static
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: ''
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_label_component
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
        - __meta_kubernetes_pod_container_name
        target_label: __path__
kind: ConfigMap
metadata:
  name: promtail-configmap
  namespace: monitoring
  resourceVersion: '17042330'

3）启动deployment

bash 复制代码 
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f promtailServiceAccount.yaml
root@iZj6c72dzbei17o2cuksmeZ:~/loki# kubectl apply -f promtail.yaml

3、web界面操作

1）登录web界面，导入loki数据源

点击Data sources --> +Add new data source

选择loki项

自定义相关的参数项

滑倒最下面点击保存

2）自定义条件查看日志信息

3）导入Dashboards

选中Dashboards-->New-->import

输入模块id-->加载

自定义Dashboard名称--id--选择loki数据源

最后点击导入

相关推荐
企鹅侠客12 分钟前
Prometheus operator怎么添加targets和告警规则
运维·云原生·kubernetes·prometheus·pod
wu8587734571 天前
【实战指南】Spring Boot + Grafana 实时监控API请求与异常，让系统问题无处可藏
spring boot·grafana
木二1 天前
附042.Kubernetes_v1.32.3生成环境高可用部署
云原生·kubernetes·prometheus·ingress·longhorn
时空无限2 天前
grafana 配置页面告警
grafana
维C°2 天前
Grafana-查询和转换数据
grafana
树下一少年2 天前
docker-compose部署prometheus+grafana+node_exporter
docker·json·grafana·prometheus·node_exporter
饭桶也得吃饭6 天前
Docker Compose 部署 Loki
docker·loki
云上艺旅8 天前
K8S学习之基础四十一：Prometheus基于Pushgateway采集数据
学习·云原生·容器·kubernetes·prometheus
云上艺旅8 天前
K8S学习之基础三十八：Kube-static-metrics监控
学习·云原生·容器·kubernetes·prometheus
热门推荐
01我决定放弃搞 Java 了02DeepSeek各版本说明与优缺点分析03如何在WPS和Word/Excel中直接使用DeepSeek功能04从零安装 LLaMA-Factory 微调 Qwen 大模型成功及所有的坑05苍穹外卖面试总结06DeepSeek R1本地化部署 Ollama + Chatbox 打造最强 AI 工具07RAG 实践- Ollama+RagFlow 部署本地知识库08本地化部署AI知识库：基于Ollama+DeepSeek+AnythingLLM保姆级教程09如何本地部署AI智能体平台，带你手搓一个AI Agent10论文笔记：交替单模态适应的多模态表征学习