每日一Go-74、Go 云原生可观测性实战之OpenTelemetry 全链路采集：Trace + Metrics + Logs

本文的前提是已经安装好了Kind和kube-prometheus-stack，如果没有安装，请参考每日一Go-70、Prometheus + Grafana 从采集到告警的完整实战（Go + Kind）

一、核心架构思路

Gin App：使用OpenTelemetry SDK 注入中间件。
OTEL Collector：作为中转站，接收Trace/Metrics/Logs，再分发给Prometheus和相关后端。
Prometheus：接收指标。
Jaeger/Tempo：接收链路数据。

二、依赖安装

安装jaeger

cpp 复制代码

kubectl create namespace monitoring
helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
helm install jaeger jaegertracing/jaeger --namespace monitoring

otel-collector-conf.yaml

apache 复制代码

apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-conf
  namespace: monitoring
data:
  otel-collector-config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
    processors:
      batch:
    exporters:
      prometheus:
        endpoint: "0.0.0.0:8889"
        resource_to_telemetry_conversion:
          enabled: true # 将资源属性转换为指标标签（推荐）
      otlp/jaeger:
        endpoint: "jaeger.monitoring.svc.cluster.local:4317"
        tls:
          insecure: true
    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [batch]
          exporters: [otlp/jaeger]
        metrics:
          receivers: [otlp]
          processors: [batch]
          exporters: [prometheus]

otel-collector-all-in-one.yaml

makefile 复制代码

---
# 1. 命名空间（假设你使用的是 monitoring，如不存在请创建）
# apiVersion: v1
# kind: Namespace
# metadata:
#   name: monitoring
---
# 2. RBAC 权限（允许 Collector 获取集群元数据）
apiVersion: v1
kind: ServiceAccount
metadata:
  name: otel-collector
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-collector-role
rules:
- apiGroups: [""]
  resources: ["pods", "namespaces", "nodes"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-collector-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: otel-collector-role
subjects:
- kind: ServiceAccount
  name: otel-collector
  namespace: monitoring
---
# 3. Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector
  namespace: monitoring
  labels:
    app: otel-collector
spec:
  replicas: 1
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      serviceAccountName: otel-collector
      containers:
      - name: otel-collector
        image: otel/opentelemetry-collector-contrib:0.90.0 # 使用 contrib 版本，功能更全
        args: ["--config=/etc/otel-collector-config.yaml"]
        ports:
        - containerPort: 4317 # OTLP gRPC
        - containerPort: 4318 # OTLP HTTP
        - containerPort: 8889 # Prometheus Exporter
        volumeMounts:
        - name: otel-collector-config-vol
          mountPath: /etc/otel-collector-config.yaml
          subPath: otel-collector-config.yaml
      volumes:
      - name: otel-collector-config-vol
        configMap:
          name: otel-collector-conf # 引用上一个回答中创建的 ConfigMap
---
# 4. Service (供 Gin 应用连接)
apiVersion: v1
kind: Service
metadata:
  name: otel-collector
  namespace: monitoring
spec:
  type: ClusterIP
  ports:
  - name: otlp-grpc
    port: 4317
    targetPort: 4317
  - name: otlp-http
    port: 4318
    targetPort: 4318
  - name: metrics
    port: 8889
    targetPort: 8889
  selector:
    app: otel-collector

三、Gin代码实现

封装otel

go 复制代码

package main
import (
    "context"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
)
// initProvider 初始化 OpenTelemetry 的 Trace 和 Metric Provider
// 返回一个清理函数，用于在程序退出时关闭 provider
func initProvider() func(context.Context) {
    ctx := context.Background()
    // 这里的地址对应 K8s Service 名
    collectorAddr := "otel-collector.monitoring.svc.cluster.local:4317"
    // 创建资源，设置服务名称为 "gin-otel-demo"
    res, _ := resource.New(ctx, resource.WithAttributes(semconv.ServiceNameKey.String("gin-otel-demo")))
    // 1. 初始化 Trace Provider
    // 创建 OTLP gRPC 导出器，将 trace 数据发送到 OpenTelemetry Collector
    traceExporter, _ := otlptracegrpc.New(ctx,
        otlptracegrpc.WithInsecure(),
        otlptracegrpc.WithEndpoint(collectorAddr),
    )
    // 创建 TracerProvider，配置批处理导出器和资源
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(traceExporter),
        sdktrace.WithResource(res),
    )
    // 设置全局 TracerProvider
    otel.SetTracerProvider(tp)
    // 2. 初始化 Metric Provider
    // 创建 OTLP gRPC 指标导出器，将 metrics 数据发送到 OpenTelemetry Collector
    metricExporter, _ := otlpmetricgrpc.New(ctx,
        otlpmetricgrpc.WithInsecure(),
        otlpmetricgrpc.WithEndpoint(collectorAddr),
    )
    // 创建周期性读取器，定期从指标导出器读取数据
    reader := metric.NewPeriodicReader(metricExporter)
    // 创建 MeterProvider，配置读取器和资源
    mp := metric.NewMeterProvider(
        metric.WithReader(reader),
        metric.WithResource(res),
    )
    // 设置全局 MeterProvider
    otel.SetMeterProvider(mp)
    // 返回清理函数，用于优雅关闭
    return func(ctx context.Context) {
        _ = tp.Shutdown(ctx)
        _ = mp.Shutdown(ctx)
    }
}

main.go

go 复制代码

package main
import (
    "context"
    "net/http"
    "strconv"
    "time"
    "log/slog" // 使用 Go 官方的 slog
    "github.com/gin-gonic/gin"
    "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/trace"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
    httpTotal = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total HTTP requests",
        },
        []string{"method", "path", "status"},
    )
)
// Prometheus中间件
func prometheusMiddleware() gin.HandlerFunc {
    return func(c *gin.Context) {
        c.Next() // 先执行后续逻辑
        // 请求完成后记录指标
        status := strconv.Itoa(c.Writer.Status())
        httpTotal.WithLabelValues(c.Request.Method, c.FullPath(), status).Inc()
    }
}
func main() {
    // 初始化 OpenTelemetry Provider，并设置清理函数
    cleanup := initProvider()
    defer cleanup(context.Background())
    // 注册指标
    prometheus.MustRegister(httpTotal)
    // 创建 Gin 路由器，默认包含 Logger 和 Recovery 中间件
    r := gin.Default()
    // 使用中间件，所有路由都会被监控
    r.Use(prometheusMiddleware())
    // 使用 otelgin 中间件，这会自动为每个请求生成 Trace
    r.Use(otelgin.Middleware("my-server"))
    // 暴露 metrics 接口
    r.GET("/metrics", func(c *gin.Context) {
        promhttp.Handler().ServeHTTP(c.Writer, c.Request)
    })
    // 定义 GET /user/:id 路由
    r.GET("/user/:id", func(c *gin.Context) {
        // 1. 获取当前请求的 Span（由 otelgin 中间件创建）
        span := trace.SpanFromContext(c.Request.Context())
        traceID := span.SpanContext().TraceID().String()
        // 2. Metrics 埋点示例：使用 OpenTelemetry Metrics API
        meter := otel.Meter("gin-server")
        counter, _ := meter.Int64Counter("api_requests_total")
        // 增加计数器，并添加 path 属性
        counter.Add(c.Request.Context(), 1, metric.WithAttributes(attribute.String("path", "/user")))
        // 3. Logs 实战：手动注入 TraceID
        // 在实际项目中，建议写一个 slog 的自定义 Handler 自动提取
        logger := slog.With("trace_id", traceID)
        logger.Info("开始处理用户请求", "user_id", c.Param("id"))
        // 模拟业务逻辑处理
        time.Sleep(100 * time.Millisecond)
        // 如果用户 ID 为 "error"，返回 404 错误
        if c.Param("id") == "error" {
            // 设置 span 属性，标记为错误
            span.SetAttributes(attribute.Bool("error", true))
            logger.Error("用户不存在")
            c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
            return
        }
        // 返回成功响应，包含 trace_id
        c.JSON(http.StatusOK, gin.H{"status": "ok", "trace_id": traceID})
    })
    // 启动 HTTP 服务器，监听 8080 端口
    r.Run(":8080")
}

打包镜像

bash 复制代码

docker build -t imoowi/golang_per_day:day74 .

kind load docker-image imoowi/golang_per_day:day74 --name golang-per-day

app.yaml

makefile 复制代码

apiVersion: apps/v1
kind: Deployment
metadata:
  name: gin-app
  namespace: codee-jun
spec:
  replicas: 1
  selector:
    matchLabels:
      app: gin-app
  template:
    metadata:
      labels:
        app: gin-app
    spec:
      containers:
      - name: gin-app
        image: imoowi/golang_per_day:day74
        imagePullPolicy: IfNotPresent # 优先使用本地 load 进去的镜像
        ports:
        - containerPort: 8080
        env:
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
          value: "otel-collector.monitoring.svc.cluster.local:4317"
---
apiVersion: v1
kind: Service
metadata:
  name: gin-app
  namespace: codee-jun
  labels:
    app: gin-app  # ✅ 给 Service 打 label（必须）
spec:
  selector:
    app: gin-app
  ports:
  - name: http
    port: 8080
    targetPort: 8080

otel-collector-monitor.yaml

makefile 复制代码

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: gin-app-monitor
  namespace: monitoring
  labels:
    release: monitoring
spec:
  namespaceSelector:
    matchNames:
    - codee-jun
  selector:
    matchLabels:
      app: gin-app
  endpoints:
  - port: http
    path: /metrics
    interval: 15s

四、发布app

nginx 复制代码

kubectl apply -f app.yaml

apache 复制代码

kubectl get pods -n codee-jun                                                                            
NAME                      READY   STATUS    RESTARTS   AGE                                                                         
gin-app-8bd6b88c5-59xt6   1/1     Running   0          6s

sql 复制代码

kubectl logs -f gin-app-8bd6b88c5-59xt6 -n codee-jun                                                     
[GIN-debug] [WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached.                        

[GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production.                                             
 - using env:   export GIN_MODE=release                                                                                            
 - using code:  gin.SetMode(gin.ReleaseMode)                                                                                       

[GIN-debug] GET    /metrics                  --> main.main.func1 (5 handlers)                                                      
[GIN-debug] GET    /user/:id                 --> main.main.func2 (5 handlers)                                                      
[GIN-debug] [WARNING] You trusted all proxies, this is NOT safe. We recommend you to set a value.                                  
Please check https://github.com/gin-gonic/gin/blob/master/docs/doc.md#dont-trust-all-proxies for details.                          
[GIN-debug] Listening and serving HTTP on :8080                                                                                    
[GIN] 2026/01/17 - 15:50:09 | 200 |     711.625µs |     10.244.1.36 | GET      "/metrics"                                          
[GIN] 2026/01/17 - 15:50:24 | 200 |    2.729667ms |     10.244.1.36 | GET      "/metrics"

五、制造流量

sql 复制代码

kubectl exec -it gin-app-8bd6b88c5-vtlx9 -n codee-jun -- sh

makefile 复制代码

 # apk add curl                                                                                                                   
( 1/10) Installing brotli-libs (1.2.0-r0)                                                                                          
( 2/10) Installing c-ares (1.34.6-r0)                                                                                              
( 3/10) Installing libunistring (1.4.1-r0)                                                                                         
( 4/10) Installing libidn2 (2.3.8-r0)                                                                                              
( 5/10) Installing nghttp2-libs (1.68.0-r0)                                                                                        
( 6/10) Installing nghttp3 (1.13.1-r0)                                                                                             
( 7/10) Installing libpsl (0.21.5-r3)                                                                                              
( 8/10) Installing zstd-libs (1.5.7-r2)                                                                                            
( 9/10) Installing libcurl (8.17.0-r1)                                                                                             
(10/10) Installing curl (8.17.0-r1)                                                                                                
Executing busybox-1.37.0-r30.trigger

cpp 复制代码

~ # curl http://localhost:8080/user/1                                                                                              
{"status":"ok","trace_id":"a502e9ff9a293fa0c26d25030aa28f49"}

bash 复制代码

kubectl logs -l app=otel-collector -n monitoring -f   
2026-01-17T16:07:14.065Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:07:28.972Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:07:44.071Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:07:59.165Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:08:14.065Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:08:29.158Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:08:44.056Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:08:59.141Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:09:14.035Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:09:29.137Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:09:44.223Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}                                                                                            
2026-01-17T16:09:59.145Z        info    exporterhelper/retry_sender.go:120      Exporting finished.     {"kind": "exporter", "data_
type": "traces", "name": "otlp/jaeger"}

六、访问jaeger

apache 复制代码

kubectl port-forward svc/jaeger -n monitoring 16686:16686

七、访问prometheus

apache 复制代码

kubectl port-forward svc/monitoring-kube-prometheus-prometheus 9090 -n monitoring

友情链接:加班费计算器(vx小程序搜索"加班计")

*源码地址*

私信给

如果您喜欢这篇文章，请您（点赞、分享、亮爱心），万分感谢！