k8s部署grafana beyla实现app应用服务依赖图可观测

k8s部署grafana beyla

OS:

Static hostname: test

Icon name: computer-vm

Chassis: vm

Machine ID: 22349ac6f9ba406293d0541bcba7c05d

Boot ID: 83bb7e5dbf27453c94ff9f1fe88d5f02

Virtualization: vmware

Operating System: Ubuntu 22.04.4 LTS

Kernel: Linux 5.15.0-105-generic

Architecture: x86-64

Hardware Vendor: VMware, Inc.

Hardware Model: VMware Virtual Platform

kubespray version:

2.25.0

kubernetes version:

1.29.5

部署测试用nginx

bash 复制代码
cat > nginx.yaml <<EOF
kind: Deployment
apiVersion: apps/v1
metadata:
  name: docs
spec:
  replicas: 2
  selector:
    matchLabels:
      app: docs
  template:
    metadata:
      labels:
        app: docs
    spec:
      containers:
        - name: docs-server
          image: httpd:latest
          ports:
            - containerPort: 80
              protocol: TCP
              name: http
---
apiVersion: v1
kind: Service
metadata:
  name: docs
spec:
  selector:
    app: docs
  ports:
    - protocol: TCP
      port: 80
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: website
spec:
  replicas: 2
  selector:
    matchLabels:
      app: website
  template:
    metadata:
      labels:
        app: website
    spec:
      containers:
        - name: website-server
          image: dockerhub.timeweb.cloud/httpd:latest
          ports:
            - containerPort: 80
              protocol: TCP
              name: http
---
apiVersion: v1
kind: Service
metadata:
  name: website
spec:
  selector:
    app: website
  ports:
    - protocol: TCP
      port: 80
EOF
# 创建
kubectl apply -f nginx.yaml
# 转发端口
kubectl port-forward services/website 8080:80
kubectl port-forward services/docs 8081:80

部署grafana beyla

bash 复制代码
# 创建命名空间
kubectl create namespace beyla
# 创建serviceaccount
cat > beyla-serviceaccount.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: beyla
  name: beyla
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: beyla
rules:
  - apiGroups: ["apps"]
    resources: ["replicasets"]
    verbs: ["list", "watch"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: beyla
subjects:
  - kind: ServiceAccount
    name: beyla
    namespace: beyla
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: beyla
EOF

kubectl apply -f beyla-serviceaccount.yaml

cat > beyla.yaml <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: beyla
  name: beyla-config
data:
  beyla-config.yml: |
    # this is required to enable kubernetes discovery and metadata
    attributes:
      kubernetes:
        enable: true
    # this will provide automatic routes report while minimizing cardinality
    routes:
      unmatched: heuristic
    # let's instrument only the docs server
    discovery:
      services:
        - k8s_deployment_name: "^docs$"
        # uncomment the following line to also instrument the website server
        # - k8s_deployment_name: "^website$"
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  namespace: beyla
  name: beyla
spec:
  selector:
    matchLabels:
      instrumentation: beyla
  template:
    metadata:
      labels:
        instrumentation: beyla
    spec:
      serviceAccountName: beyla
      hostPID: true # mandatory!
      containers:
        - name: beyla
          image: dockerhub.timeweb.cloud/grafana/beyla:1.2
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true # mandatory!
            readOnlyRootFilesystem: true
          volumeMounts:
            - mountPath: /config
              name: beyla-config
            - mountPath: /var/run/beyla
              name: var-run-beyla
          env:
            - name: BEYLA_CONFIG_PATH
              value: "/config/beyla-config.yml"
            - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
              value: "http://10.1.1.71:4318/v1/traces"
            - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL
              value: "http/protobuf"
#            - name: OTEL_EXPORTER_OTLP_HEADERS
#              valueFrom:
#                secretKeyRef:
#                  name: grafana-credentials
#                  key: otlp-headers
      volumes:
        - name: beyla-config
          configMap:
            name: beyla-config
        - name: var-run-beyla
          emptyDir: {}
EOF

kubectl apply -f beyla.yaml

安装grafana

bash 复制代码
apt-get install -y adduser libfontconfig1 musl
wget https://dl.grafana.com/oss/release/grafana_10.4.2_amd64.deb
dpkg -i grafana_10.4.2_amd64.deb
systemctl start grafana-server
systemctl enable grafana-server

安装prometheus

bash 复制代码
wget --no-check-certificate https://github.com/prometheus/prometheus/releases/download/v2.45.4/prometheus-2.45.4.linux-amd64.tar.gz
tar -zxf prometheus-2.45.4.linux-amd64.tar.gz
mkdir -p /etc/prometheus
mkdir -p /export/prometheus/data
cp -r prometheus-2.45.4.linux-amd64/* /etc/prometheus/
mv /etc/prometheus/prometheus /usr/local/bin/
mv /etc/prometheus/promtool /usr/local/bin/

# 配置抓取promttheus
cat <<EOF >/etc/prometheus/prometheus.yml
global:
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"
scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]
  - job_name: "beyla"
    static_configs:
      - targets: ["localhost:9101", "localhost:9102", "localhost:9103"]
EOF
# 启动
# 使用--web.enable-remote-write-receiver启用远程写入接口来接收tempo的service graph数据,地址为/api/v1/write
screen -dmS prom prometheus --config.file=/etc/prometheus/prometheus.yml --web.enable-remote-write-receiver --storage.tsdb.path=/export/prometheus/data --web.console.libraries=/etc/prometheus/console_libraries --web.console.templates=/etc/prometheus/consoles --storage.tsdb.retention=7d &

安装tempo

tempo部署
tempo配置

安装

bash 复制代码
curl -Lo tempo_2.4.1_linux_amd64.deb https://github.com/grafana/tempo/releases/download/v2.4.1/tempo_2.4.1_linux_amd64.deb
echo 2fdd167cbb00d732435123a254469ec4cfde3c525a4ec89d235423a5e9abc4b3 \
  tempo_2.4.1_linux_amd64.deb | sha256sum -c
dpkg -i tempo_2.4.1_linux_amd64.deb

配置

bash 复制代码
cat > /etc/tempo/config.yml <<EOF
server:
  http_listen_port: 3200

distributor:
  receivers:
      otlp:
        protocols:
          http:
          grpc:

compactor:
  compaction:
    block_retention: 48h

metrics_generator:
  registry:
    external_labels:
      source: tempo
      cluster: linux-microservices
  storage:
    path: /tmp/tempo/generator/wal
    remote_write:
    - url: http://localhost:9090/api/v1/write
      send_exemplars: true

storage:
#   trace:
#     backend: s3
#     s3:
#       endpoint: s3.us-east-1.amazonaws.com
#       bucket: grafana-traces-data
#       forcepathstyle: true
#       # set to false if endpoint is https
#       insecure: true
#       access_key: # TODO - Add S3 access key
#       secret_key: # TODO - Add S3 secret key

  trace:
    backend: local
    wal:
      path: /tmp/tempo/wal
    local:
      path: /tmp/tempo/blocks
overrides:
  defaults:
    metrics_generator:
      processors: [service-graphs, span-metrics]
EOF

启动

bash 复制代码
systemctl start tempo.service
systemctl enable tempo.service
systemctl is-active tempo

生成trace数据

访问生成trace信息

bash 复制代码
curl http://localhost:8080
curl http://localhost:8080/foo
curl http://localhost:8081
curl http://localhost:8081/foo

tail pod日志

bash 复制代码
kubectl logs -f beyla-spb9s -n beyla

grafana面板配置

添加tempo源并启用node graph


启用service graph

prometheus需要使用--web.enable-remote-write-receiver启用远程写入接口来接收tempo的service graph数据,地址为/api/v1/write
prometheus http API

在explorer中搜索service graph

设置hosts

bash 复制代码
# vim /etc/hosts
127.0.0.1 prometheus
相关推荐
时迁24729 分钟前
【k8s】k8s是怎么实现自动扩缩的
云原生·容器·kubernetes·k8s
诡异森林。4 小时前
Docker--Docker网络原理
网络·docker·容器
matrixlzp5 小时前
K8S Service 原理、案例
云原生·容器·kubernetes
angushine6 小时前
让Docker端口映射受Firewall管理而非iptables
运维·docker·容器
SimonLiu0098 小时前
清理HiNas(海纳斯) Docker日志并限制日志大小
java·docker·容器
高峰君主10 小时前
Docker容器持久化
docker·容器·eureka
能来帮帮蒟蒻吗11 小时前
Docker安装(Ubuntu22版)
笔记·学习·spring cloud·docker·容器
言之。15 小时前
别学了,打会王者吧
java·python·mysql·容器·spark·php·html5
秦始皇爱找茬18 小时前
docker部署Jenkins工具
docker·容器·jenkins
樽酒ﻬق21 小时前
Kubernetes 常用运维命令整理
运维·容器·kubernetes