33.5 remote实战项目之设计prometheus数据源的结构

福大大架构师每日一题2024-12-05 9:01

本节重点介绍 :

  • 项目要求
    • 通过remote read读取prometheus中的数据
    • 通过remote write向prometheus中写入数据
  • 准备工作
    • 新建项目 prome_remote_read_write
    • 设计prometheus 数据源的结构
    • 初始化

项目要求

  • 通过remote read读取prometheus中的数据
  • 通过remote write向prometheus中写入数据

准备工作

新建项目 prome_remote_read_write

shell 复制代码 
go mod init prome_remote_read_write

准备配置文件 prome_remote_read_write.yml

  • remoteWrite代表 支持remote_write的多个后端
  • remoteRead代表 支持remote_read的多个后端
yaml 复制代码 
remoteWrite:
  # m3db的配置
  #- name: m3db01
  #  url: http://localhost:7201/api/v1/prom/remote/write
  #  remoteTimeoutSecond: 5

  # prometheus的配置
  - name: prome01
    url: http://172.20.70.205:9090/api/v1/write
    remoteTimeoutSecond: 5
remoteRead:
  - name: prome01
    url: http://172.20.70.205:9090/api/v1/read
    remoteTimeoutSecond: 5

配置文件解析

  • config/config.go
go 复制代码 
package config

import (
	"github.com/toolkits/pkg/logger"
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

type RemoteConfig struct {
	Name                string `yaml:"name"`
	Url                 string `yaml:"url"`
	RemoteTimeoutSecond int    `yaml:"remoteTimeoutSecond"`
}

type PromeSection struct {
	RemoteWrite []RemoteConfig `yaml:"remoteWrite"`
	RemoteRead  []RemoteConfig `yaml:"remoteRead"`
}

func Load(s string) (*PromeSection, error) {
	cfg := &PromeSection{}

	err := yaml.Unmarshal([]byte(s), cfg)

	if err != nil {
		return nil, err
	}
	return cfg, nil
}

func LoadFile(filename string) (*PromeSection, error) {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	cfg, err := Load(string(content))
	if err != nil {
		logger.Errorf("[parsing YAML file errr...][error:%v]", err)
		return nil, err
	}
	return cfg, nil
}

main.go中解析配置

go 复制代码 
package main

import (
	"flag"
	"github.com/toolkits/pkg/logger"
	"math/rand"
	"prome_remote_read_write/config"
	"prome_remote_read_write/datasource"
	"time"
)

func main() {
	rand.Seed(time.Now().UnixNano())
	configFile := flag.String("config", "prome_remote_read_write.yml",
		"Address on which to expose metrics and web interface.")
	flag.Parse()

	sConfig, err := config.LoadFile(*configFile)
	if err != nil {
		logger.Infof("config.LoadFile Error,Exiting ...error:%v", err)
		return
	}
}

设计prometheus 数据源的结构

  • 位置 datasource/prome.go
go 复制代码 
package datasource

import (
	"github.com/go-kit/kit/log"
	"github.com/prometheus/client_golang/prometheus"
	config_util "github.com/prometheus/common/config"
	"github.com/prometheus/common/model"
	"github.com/prometheus/common/promlog"
	pc "github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/prompb"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/toolkits/pkg/logger"
	"go.uber.org/atomic"
	"io/ioutil"
	"net/http"
	"net/url"
	"prome_remote_read_write/config"
	"time"
)

type PromeDataSource struct {
	Section      *config.PromeSection            //配置
	PushQueue    chan []prompb.TimeSeries        // 数据推送的chan
	LocalTmpDir  string                          // 本地临时目录，存放queries.active文件
	Queryable    storage.SampleAndChunkQueryable // 除了promql的查询，需要后端存储，如查询series
	QueryEngine  *promql.Engine                  // promql相关查询
	WriteTargets []*HttpClient                   // remote_write写入的后端地址
}

type HttpClient struct {
	remoteName string // Used to differentiate clients in metrics.
	url        *url.URL
	Client     *http.Client
	timeout    time.Duration
}

new函数

  • 根据传入的配置new
go 复制代码 
func NewPromeDataSource(cg *config.PromeSection) *PromeDataSource {
	pd := &PromeDataSource{
		Section:   cg,
		PushQueue: make(chan []prompb.TimeSeries, 10000),
	}
	return pd
}

Init初始化函数

  • 完整代码如下
go 复制代码 
type safePromQLNoStepSubqueryInterval struct {
	value atomic.Int64
}

func durationToInt64Millis(d time.Duration) int64 {
	return int64(d / time.Millisecond)
}
func (i *safePromQLNoStepSubqueryInterval) Set(ev model.Duration) {
	i.value.Store(durationToInt64Millis(time.Duration(ev)))
}
func (i *safePromQLNoStepSubqueryInterval) Get(int64) int64 {
	return i.value.Load()
}

func NewPromeDataSource(cg *config.PromeSection) *PromeDataSource {
	pd := &PromeDataSource{
		Section:   cg,
		PushQueue: make(chan []prompb.TimeSeries, 10000),
	}
	return pd
}

func (pd *PromeDataSource) Init() {
	// 模拟创建本地存储目录
	dbDir, err := ioutil.TempDir("", "tsdb-api-ready")
	if err != nil {
		logger.Errorf("[error_create_local_tsdb_dir][err: %v]", err)
		return
	}
	pd.LocalTmpDir = dbDir
	promlogConfig := promlog.Config{}
	// 使用本地目录创建remote-storage
	remoteS := remote.NewStorage(promlog.New(&promlogConfig), prometheus.DefaultRegisterer, func() (int64, error) {
		return 0, nil
	}, dbDir, 1*time.Minute, nil)

	// ApplyConfig 加载queryables
	remoteReadC := make([]*pc.RemoteReadConfig, 0)
	for _, u := range pd.Section.RemoteRead {

		ur, err := url.Parse(u.Url)
		if err != nil {
			logger.Errorf("[prome_ds_init_error][parse_url_error][url:%+v][err:%+v]", u.Url, err)
			continue
		}

		remoteReadC = append(remoteReadC,
			&pc.RemoteReadConfig{
				URL:           &config_util.URL{URL: ur},
				RemoteTimeout: model.Duration(time.Duration(u.RemoteTimeoutSecond) * time.Second),
				ReadRecent:    true,
			},
		)
	}
	if len(remoteReadC) == 0 {
		logger.Errorf("[prome_ds_error_got_zero_remote_read_storage]")
		return
	}
	err = remoteS.ApplyConfig(&pc.Config{RemoteReadConfigs: remoteReadC})
	if err != nil {
		logger.Errorf("[error_load_remote_read_config][err: %v]", err)
		return
	}
	pLogger := log.NewNopLogger()

	noStepSubqueryInterval := &safePromQLNoStepSubqueryInterval{}

	queryQueueDir, err := ioutil.TempDir(dbDir, "prom_query_concurrency")
	opts := promql.EngineOpts{
		Logger:                   log.With(pLogger, "component", "query engine"),
		Reg:                      prometheus.DefaultRegisterer,
		MaxSamples:               50000000,
		Timeout:                  30 * time.Second,
		ActiveQueryTracker:       promql.NewActiveQueryTracker(queryQueueDir, 20, log.With(pLogger, "component", "activeQueryTracker")),
		LookbackDelta:            5 * time.Minute,
		NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get,
		EnableAtModifier:         true,
	}

	queryEngine := promql.NewEngine(opts)
	pd.QueryEngine = queryEngine
	pd.Queryable = remoteS

	// 初始化writeClients
	if len(pd.Section.RemoteWrite) == 0 {
		logger.Warningf("[prome_ds_init_with_zero_RemoteWrite_target]")
		logger.Infof("[successfully_init_prometheus_datasource][remote_read_num:%+v][remote_write_num:%+v]",
			len(pd.Section.RemoteRead),
			len(pd.Section.RemoteWrite),
		)
		return
	}
	writeTs := make([]*HttpClient, 0)
	for _, u := range pd.Section.RemoteWrite {
		ur, err := url.Parse(u.Url)
		if err != nil {
			logger.Errorf("[prome_ds_init_error][parse_url_error][url:%+v][err:%+v]", u.Url, err)
			continue
		}
		writeTs = append(writeTs,
			&HttpClient{
				remoteName: u.Name,
				url:        ur,
				Client:     &http.Client{},
				timeout:    time.Duration(u.RemoteTimeoutSecond) * time.Second,
			})
	}
	pd.WriteTargets = writeTs
	// 开启prometheus 队列消费协程
	go pd.remoteWrite()
	logger.Infof("[successfully_init_prometheus_datasource][remote_read_num:%+v][remote_write_num:%+v]",
		len(remoteReadC),
		len(writeTs),
	)
}

创建本地存储目录和remote-storage

  • 模拟创建本地存储目录
go 复制代码 
	// 模拟创建本地存储目录
	dbDir, err := ioutil.TempDir("", "tsdb-api-ready")
	if err != nil {
		logger.Errorf("[error_create_local_tsdb_dir][err: %v]", err)
		return
	}
	pd.LocalTmpDir = dbDir
  • 使用本地目录创建remote-storage
go 复制代码 
	// 使用本地目录创建remote-storage
	remoteS := remote.NewStorage(promlog.New(&promlogConfig), prometheus.DefaultRegisterer, func() (int64, error) {
		return 0, nil
	}, dbDir, 1*time.Minute, nil)

创建remote_read对象

  • 遍历配置中的remote_read，构造RemoteReadConfig
  • 使用RemoteReadConfig.ApplyConfig 生效配置
go 复制代码 
	// ApplyConfig 加载queryables
	remoteReadC := make([]*pc.RemoteReadConfig, 0)
	for _, u := range pd.Section.RemoteRead {

		ur, err := url.Parse(u.Url)
		if err != nil {
			logger.Errorf("[prome_ds_init_error][parse_url_error][url:%+v][err:%+v]", u.Url, err)
			continue
		}

		remoteReadC = append(remoteReadC,
			&pc.RemoteReadConfig{
				URL:           &config_util.URL{URL: ur},
				RemoteTimeout: model.Duration(time.Duration(u.RemoteTimeoutSecond) * time.Second),
				ReadRecent:    true,
			},
		)
	}
	if len(remoteReadC) == 0 {
		logger.Errorf("[prome_ds_error_got_zero_remote_read_storage]")
		return
	}
	err = remoteS.ApplyConfig(&pc.Config{RemoteReadConfigs: remoteReadC})
	if err != nil {
		logger.Errorf("[error_load_remote_read_config][err: %v]", err)
		return
	}

创建QueryEngine并赋值

go 复制代码 
	noStepSubqueryInterval := &safePromQLNoStepSubqueryInterval{}

	queryQueueDir, err := ioutil.TempDir(dbDir, "prom_query_concurrency")
	opts := promql.EngineOpts{
		Logger:                   log.With(pLogger, "component", "query engine"),
		Reg:                      prometheus.DefaultRegisterer,
		MaxSamples:               50000000,
		Timeout:                  30 * time.Second,
		ActiveQueryTracker:       promql.NewActiveQueryTracker(queryQueueDir, 20, log.With(pLogger, "component", "activeQueryTracker")),
		LookbackDelta:            5 * time.Minute,
		NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get,
		EnableAtModifier:         true,
	}

	queryEngine := promql.NewEngine(opts)
	pd.QueryEngine = queryEngine
	pd.Queryable = remoteS

初始化writeClients创建RemoteWrite对象

  • 遍历RemoteWrite配置创建
  • 开启prometheus 队列消费协程
go 复制代码 
	// 初始化writeClients
	if len(pd.Section.RemoteWrite) == 0 {
		logger.Warningf("[prome_ds_init_with_zero_RemoteWrite_target]")
		logger.Infof("[successfully_init_prometheus_datasource][remote_read_num:%+v][remote_write_num:%+v]",
			len(pd.Section.RemoteRead),
			len(pd.Section.RemoteWrite),
		)
		return
	}
	writeTs := make([]*HttpClient, 0)
	for _, u := range pd.Section.RemoteWrite {
		ur, err := url.Parse(u.Url)
		if err != nil {
			logger.Errorf("[prome_ds_init_error][parse_url_error][url:%+v][err:%+v]", u.Url, err)
			continue
		}
		writeTs = append(writeTs,
			&HttpClient{
				remoteName: u.Name,
				url:        ur,
				Client:     &http.Client{},
				timeout:    time.Duration(u.RemoteTimeoutSecond) * time.Second,
			})
	}
	pd.WriteTargets = writeTs
	// 开启prometheus 队列消费协程
	go pd.remoteWrite()
	logger.Infof("[successfully_init_prometheus_datasource][remote_read_num:%+v][remote_write_num:%+v]",
		len(remoteReadC),
		len(writeTs),
	)

本节重点总结 :

  • 项目要求
    • 通过remote read读取prometheus中的数据
    • 通过remote write向prometheus中写入数据
  • 准备工作
    • 新建项目 prome_remote_read_write
    • 设计prometheus 数据源的结构
    • 初始化
相关推荐
Clown9521 小时前
go-zero(十九)使用Prometheus监控ES指标
elasticsearch·golang·prometheus
奔跑中的小象1 天前
基于 nvitop+Prometheus+Grafana 的物理资源与 VLLM 引擎服务监控方案
grafana·prometheus·vllm·nvitop
小黑_深呼吸1 天前
Prometheus实战教程：k8s平台-Mysql监控案例
运维·学习·kubernetes·prometheus
艾文伯特2 天前
Prometheus+Grafana+AlertManager完整安装过程
grafana·prometheus
一个向上的运维者8 天前
Prometheus生产实战全流程详解（存储/负载/调度篇）
云原生·prometheus
hwj运维之路8 天前
k8s监控方案实践（一）：部署Prometheus与Node Exporter
容器·kubernetes·prometheus
Mr.小怪8 天前
自定义prometheus exporter实现监控阿里云RDS
阿里云·adb·prometheus
阿里云云原生8 天前
剑指大规模 AI 可观测，阿里云 Prometheus 2.0 应运而生
人工智能·阿里云·prometheus
小黑_深呼吸9 天前
Prometheus实战教程：k8s平台-Redis监控案例
运维·kubernetes·prometheus·监控
码上淘金10 天前
【Prometheus】业务指标与基础指标的标签来源差异及设计解析（扩展版）
prometheus
热门推荐
01KGG转MP3工具|非KGM文件|解密音频02YOLOv8入门 | 重要性能衡量指标、训练结果评价及分析及影响mAP的因素【发论文关注的指标】03从零安装 LLaMA-Factory 微调 Qwen 大模型成功及所有的坑04【SpeedAI科研小助手】2分钟极速解决知网维普重复率、AIGC率过高，一键全文降！文件格式不变，公式都保留的！05DeepSeek各版本说明与优缺点分析06Coze扣子平台完整体验和实践（附国内和国际版对比）07Ubuntu24.04安装中文输入法08YOLOv5改进 | 添加CA注意力机制 + 增加预测层 + 更换损失函数之GIoU09苍穹外卖面试总结10第一届 “帕鲁杯“ writeup