OTel微服务链路追踪解决方案

介绍

OpenTelemetry 是 CNCF 的一个可观测性项目，旨在提供可观测性领域的标准化方案，解决观测数据的数据模型、采集、处理、导出等的标准化问题，提供与三方 vendor 无关的服务。

启动jaeger容器

docker run -d --name jaeger \
   -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
   -e COLLECTOR_OTLP_ENABLED=true      \
   -p 6831:6831/udp \
   -p 6832:6832/udp \
   -p 5778:5778 \
   -p 16686:16686 \
   -p 4317:4317  \
   -p 4318:4318  \
   -p 14250:14250 \
   -p 14268:14268 \
   -p 14269:14269 \
   -p 9411:9411 \
   jaegertracing/all-in-one
   
   
 # 参数说明
 #COLLECTOR_ZIPKIN_HOST_PORT=:9411  采集zipkin
 #COLLECTOR_OTLP_ENABLED=true   支持OTLP

容器暴露端口说明

容器暴露以下端口:

Port	Protocol	Component	Function
5775	UDP	agent	accept zipkin.thrift over compact thrift protocol (deprecated, used by legacy clients only)
6831	UDP	agent	accept jaeger.thrift over compact thrift protocol
6832	UDP	agent	accept jaeger.thrift over binary thrift protocol
5778	HTTP	agent	serve configs
16686	HTTP	query	serve frontend
14268	HTTP	collector	accept jaeger.thrift directly from clients
14250	HTTP	collector	accept model.proto
9411	HTTP	collector	Zipkin compatible endpoint (optional)

查看容器

root@node1:~# docker ps
CONTAINER ID   IMAGE                      COMMAND                  CREATED         STATUS         PORTS                                                                                                                                                                                                                                                                                                                                                                                               NAMES
4f8233f7a370   jaegertracing/all-in-one   "/go/bin/all-in-one-..."   2 minutes ago   Up 2 minutes   0.0.0.0:4317-4318->4317-4318/tcp, :::4317-4318->4317-4318/tcp, 0.0.0.0:5778->5778/tcp, :::5778->5778/tcp, 0.0.0.0:9411->9411/tcp, :::9411->9411/tcp, 0.0.0.0:14250->14250/tcp, :::14250->14250/tcp, 0.0.0.0:14268-14269->14268-14269/tcp, :::14268-14269->14268-14269/tcp, 0.0.0.0:16686->16686/tcp, :::16686->16686/tcp, 5775/udp, 0.0.0.0:6831-6832->6831-6832/udp, :::6831-6832->6831-6832/udp   jaeger

访问Jaeger

访问链接 http://192.168.202.221:16686

创建go项目测试

下载依赖

go get go.opentelemetry.io/otel/sdk
go get go.opentelemetry.io/otel/exporters/jaeger

使用otel的sdk创建服务调用

package main

import (
    "context"
    "errors"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/baggage"
    "go.opentelemetry.io/otel/codes"
    "go.opentelemetry.io/otel/exporters/jaeger"
    "go.opentelemetry.io/otel/sdk/resource"
    tracesdk "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
    "log"
)

// 初始化provider
func tracerProvider(url string) (*tracesdk.TracerProvider, error) {
    // 初始化导出器，导出到jaeger
    exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url)))
    if err != nil {
       log.Fatal(err)
    }
    // 创建traceProvider
    tp :=
       tracesdk.NewTracerProvider(
          tracesdk.WithBatcher(exp),
          tracesdk.WithResource(
             // 用于打印服务的一些信息
             resource.NewWithAttributes(
                semconv.SchemaURL,
                semconv.ServiceName("jaeger-service"),  // 服务名称
                attribute.String("environment", "dev"), // 一些属性
             ),
          ),
       )
    return tp, nil
}

func main() {
    // jaeger的api地址
    url := "http://192.168.202.221:14268/api/traces"
    // 获取trace  provider
    tp, err := tracerProvider(url)
    if err != nil {
       log.Fatal(err)
    }

    // 创建一个上下文
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    // 对tp进行关闭
    defer func() {
       err := tp.Shutdown(ctx)
       if err != nil {
          log.Fatal(err)
       }
    }()

    otel.SetTracerProvider(tp)

    // 上下文中附加一些属性
    m0, _ := baggage.NewMember("data1", "value1")
    m1, _ := baggage.NewMember("data2", "value2")
    bg, err := baggage.New(m0, m1)
    if err != nil {
       return
    }

    ctx = baggage.ContextWithBaggage(ctx, bg)

    // 开启tracing
    tr := tp.Tracer("jaeger-main") // jaeger-main为lib name

    ctx, span := tr.Start(ctx, "foo")
    defer span.End() // End表示发数据给jaeger

    // 调用bar函数
    err = bar(ctx)

    if err != nil {
       span.RecordError(err)                    // 记录错误
       span.SetStatus(codes.Error, err.Error()) // 设置状态
    }
}

func bar(ctx context.Context) error {
    tr := otel.Tracer("jaeger-main")

    _, span := tr.Start(ctx, "bar")
    defer span.End()

    // 业务逻辑
    // 设置一些属性
    span.SetAttributes(attribute.Key("test").String("test"))
    err := errors.New("bar出现了错误")

    //span.RecordError()
    // 添加一个错误事件
    span.AddEvent(err.Error())
    // 设置状态
    span.SetStatus(codes.Error, err.Error())

    return err
}

在jaeger中查看

Promethes的接入

下载依赖

go get go.opentelemetry.io/otel/exporters/prometheus

go代码如下

package main

import (
    "context"
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/exporters/prometheus"
    metric2 "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/sdk/metric"
    _ "go.opentelemetry.io/otel/sdk/metric"
    "log"
    "math/rand"
    "net/http"
    "os"
    "os/signal"
)

func main() {
    ctx := context.Background()

    // 初始化Prometheus的导出器
    exp, err := prometheus.New()

    if err != nil {
       log.Fatal(err)
    }

    // 创建Provider
    provider := metric.NewMeterProvider(metric.WithReader(exp))
    meter :=
       provider.Meter("prometheus")

    // 启动指标服务
    go serveMetrics()

    // 创建指标，将应用程序指标写入
    attrs := metric2.WithAttributes(

       attribute.Key("A").String("B"),
       attribute.Key("C").String("D"),
       attribute.Key("E").String("F"),
    )

    // 计数器指标
    counter, err := meter.Float64Counter("foo", metric2.WithDescription("foo counter描述"))
    if err != nil {
       log.Fatal(err)
    }

    // 加5，自服务启动后加5
    counter.Add(ctx, 5, attrs)

    // 实时指标
    gauge, err := meter.Float64ObservableGauge("bar", metric2.WithDescription("bar observer 描述"))
    if err != nil {
       log.Fatal(err)
    }

    // 直方图，根据范围对数据进行分类归类
    histogram, err := meter.Float64Histogram("baz", metric2.WithDescription("baz histogram 描述"))

    if err != nil {
       log.Fatal(err)
    }
    // 添加数据记录
    histogram.Record(ctx, 23, attrs)
    histogram.Record(ctx, 7, attrs)
    histogram.Record(ctx, 56, attrs)
    histogram.Record(ctx, 23, attrs)
    histogram.Record(ctx, 43, attrs)
    histogram.Record(ctx, 60, attrs)
    histogram.Record(ctx, 14, attrs)
    histogram.Record(ctx, 21, attrs)
    histogram.Record(ctx, 90, attrs)
    histogram.Record(ctx, 24, attrs)
    histogram.Record(ctx, 67, attrs)

    // 注册回调函数
    _, err = meter.RegisterCallback(func(ctx context.Context, observer metric2.Observer) error {
       n := rand.Float64() * (100.)
       observer.ObserveFloat64(gauge, n, attrs)
       return nil
    }, gauge)

    ctx, _ = signal.NotifyContext(ctx, os.Interrupt)
    <-ctx.Done()
}

func serveMetrics() {
    http.Handle("/metrics", promhttp.Handler())

    err := http.ListenAndServe(":2223", nil)

    if err != nil {
       log.Fatal(err)
    }
}

查看端口数据

访问 http://192.168.0.204:2223/metrics

在Prometheus中配置文件中添加该endpoint

  - job_name: "tracing_status"
    metrics_path: /metrics
    static_configs:
      - targets: ["192.168.0.204:2223"]

重启Prometheus

systemctl restart prometheus

查看状态

在图形界面查询

在Grafana中添加图表

参考文档

• rocdu.gitbook.io/jaeger-doc-...
• www.jaegertracing.io/docs/1.50/d...
• juejin.cn/post/723371...