从零实现一个分布式监控：Prometheus的核心设计

前言

你有没有想过：在微服务架构中，你是怎么知道系统CPU飙高了、内存泄漏了、接口变慢了的？这些指标是怎么采集、存储、告警的？

Prometheus是云原生监控的事实标准，采用拉取模式采集指标。

今天我们从零实现一个分布式监控系统：

· 指标采集（Counter/Gauge/Histogram）

· 指标存储（时间序列数据库）

· 查询语言（PromQL简化版）

· 告警规则引擎

· 可视化（简单Dashboard）

一、监控系统核心原理

架构图

```

┌─────────────────────────────────────────────────────────────┐

│ 应用服务 │

│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │

│ │ 服务A │ │ 服务B │ │ 服务C │ │

│ │ 指标暴露 │ │ 指标暴露 │ │ 指标暴露 │ │

│ │ /metrics │ │ /metrics │ │ /metrics │ │

│ └─────────────┘ └─────────────┘ └─────────────┘ │

└─────────────────────────────────────────────────────────────┘

│

▼

┌─────────────────────────────────────────────────────────────┐

│ Prometheus │

│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │

│ │ 指标抓取 │→│ 时间序列 │→│ 告警评估 │ │

│ │ (Pull) │ │ 存储 │ │ (Alert) │ │

│ └─────────────┘ └─────────────┘ └─────────────┘ │

└─────────────────────────────────────────────────────────────┘

│

▼

┌─────────────┐

│ Grafana │

│ (可视化) │

└─────────────┘

```

核心概念

概念说明示例

Counter 只增不减请求总数

Gauge 可增可减 CPU使用率

Histogram 分布统计请求延迟

Label 维度标签 service, endpoint, status

二、完整代码实现

基础数据结构

```c

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#include <pthread.h>

#include <time.h>

#include <errno.h>

#include <math.h>

#define MAX_METRIC_NAME 128

#define MAX_LABELS 8

#define MAX_LABEL_KEY 32

#define MAX_LABEL_VALUE 64

#define MAX_SERIES 10000

#define MAX_HISTOGRAM_BUCKETS 20

// 指标类型

typedef enum {

METRIC_COUNTER = 0,

METRIC_GAUGE,

METRIC_HISTOGRAM,

METRIC_SUMMARY

} metric_type_t;

// 标签

typedef struct label {

char key $MAX_LABEL_KEY$ ;

char value $MAX_LABEL_VALUE$ ;

} label_t;

// 指标值

typedef struct metric_value {

metric_type_t type;

char name $MAX_METRIC_NAME$ ;

label_t labels $MAX_LABELS$ ;

int label_count;

double value;

double sum;

double count;

double buckets $MAX_HISTOGRAM_BUCKETS$ ;

double bucket_upper $MAX_HISTOGRAM_BUCKETS$ ;

int bucket_count;

time_t timestamp;

struct metric_value *next;

} metric_value_t;

// 时间序列

typedef struct time_series {

char metric_name $MAX_METRIC_NAME$ ;

label_t labels $MAX_LABELS$ ;

int label_count;

metric_value_t *values;

int value_count;

struct time_series *next;

} time_series_t;

// 告警规则

typedef struct alert_rule {

char name $64$ ;

char expr $128$ ;

double threshold;

char condition $8$ ; // >, <, >=, <=

int for_seconds;

char severity $16$ ;

char message $256$ ;

struct alert_rule *next;

} alert_rule_t;

// 告警事件

typedef struct alert_event {

char rule_name $64$ ;

char status $16$ ; // firing, resolved

time_t start_time;

time_t end_time;

char value $64$ ;

struct alert_event *next;

} alert_event_t;

// Prometheus

typedef struct prometheus {

time_series_t *series;

alert_rule_t *alert_rules;

alert_event_t *alert_events;

int retention_days;

int scrape_interval_ms;

pthread_mutex_t mutex;

int running;

pthread_t scrape_thread;

pthread_t alert_thread;

} prometheus_t;

```

指标采集

```c

// 创建Prometheus

prometheus_t *prom_create(int scrape_interval_ms, int retention_days) {

prometheus_t *p = malloc(sizeof(prometheus_t));

memset(p, 0, sizeof(prometheus_t));

p->scrape_interval_ms = scrape_interval_ms;

p->retention_days = retention_days;

p->running = 1;

pthread_mutex_init(&p->mutex, NULL);

printf(" $Prometheus$ 启动，抓取间隔: %dms\n", scrape_interval_ms);

return p;

}

// 查找时间序列

time_series_t *find_series(prometheus_t *p, const char *name,

label_t *labels, int label_count) {

time_series_t *ts = p->series;

while (ts) {

if (strcmp(ts->metric_name, name) != 0) {

ts = ts->next;

continue;

}

if (ts->label_count != label_count) {

ts = ts->next;

continue;

}

int match = 1;

for (int i = 0; i < label_count; i++) {

if (strcmp(ts->labels $i$ .key, labels $i$ .key) != 0 ||

strcmp(ts->labels $i$ .value, labels $i$ .value) != 0) {

match = 0;

break;

}

if (match) return ts;

ts = ts->next;

}

// 创建新时间序列

ts = malloc(sizeof(time_series_t));

strcpy(ts->metric_name, name);

ts->label_count = label_count;

for (int i = 0; i < label_count && i < MAX_LABELS; i++) {

strcpy(ts->labels $i$ .key, labels $i$ .key);

strcpy(ts->labels $i$ .value, labels $i$ .value);

}

ts->values = NULL;

ts->value_count = 0;

ts->next = p->series;

p->series = ts;

return ts;

}

// 添加指标值

void prom_add_metric(prometheus_t *p, const char *name, metric_type_t type,

label_t *labels, int label_count, double value) {

pthread_mutex_lock(&p->mutex);

time_series_t *ts = find_series(p, name, labels, label_count);

// 限制历史数据量

if (ts->value_count >= 10000) {

metric_value_t *old = ts->values;

ts->values = old->next;

free(old);

ts->value_count--;

}

metric_value_t *mv = malloc(sizeof(metric_value_t));

mv->type = type;

strcpy(mv->name, name);

mv->label_count = label_count;

for (int i = 0; i < label_count; i++) {

strcpy(mv->labels $i$ .key, labels $i$ .key);

strcpy(mv->labels $i$ .value, labels $i$ .value);

}

mv->value = value;

mv->timestamp = time(NULL);

mv->next = ts->values;

ts->values = mv;

ts->value_count++;

pthread_mutex_unlock(&p->mutex);

}

// 抓取目标（模拟）

void prom_scrape(prometheus_t *p, const char *target) {

// 模拟从/metrics端点抓取指标

label_t labels $2$ ;

strcpy(labels $0$ .key, "service");

strcpy(labels $0$ .value, target);

strcpy(labels $1$ .key, "env");

strcpy(labels $1$ .value, "prod");

// 模拟CPU使用率

double cpu = 30 + (rand() % 60);

prom_add_metric(p, "cpu_usage", METRIC_GAUGE, labels, 2, cpu);

// 模拟请求总数

double req = 100 + rand() % 900;

prom_add_metric(p, "http_requests_total", METRIC_COUNTER, labels, 2, req);

// 模拟内存使用

double mem = 1024 + rand() % 4096;

prom_add_metric(p, "memory_usage_mb", METRIC_GAUGE, labels, 2, mem);

printf(" $Scrape$ %s: CPU=%.1f%%, 请求=%.0f, 内存=%.0fMB\n",

target, cpu, req, mem);

}

```

告警规则引擎

```c

// 添加告警规则

void prom_add_alert_rule(prometheus_t *p, const char *name, const char *metric,

const char *condition, double threshold,

int for_seconds, const char *severity,

const char *message) {

pthread_mutex_lock(&p->mutex);

alert_rule_t *rule = malloc(sizeof(alert_rule_t));

strcpy(rule->name, name);

strcpy(rule->expr, metric);

strcpy(rule->condition, condition);

rule->threshold = threshold;

rule->for_seconds = for_seconds;

strcpy(rule->severity, severity);

strcpy(rule->message, message);

rule->next = p->alert_rules;

p->alert_rules = rule;

pthread_mutex_unlock(&p->mutex);

printf(" $告警$ 规则: %s (%s %s %.0f)\n", name, metric, condition, threshold);

}

// 检查告警条件

int check_condition(double value, const char *condition, double threshold) {

if (strcmp(condition, ">") == 0) return value > threshold;

if (strcmp(condition, "<") == 0) return value < threshold;

if (strcmp(condition, ">=") == 0) return value >= threshold;

if (strcmp(condition, "<=") == 0) return value <= threshold;

return 0;

}

// 评估告警

void prom_evaluate_alerts(prometheus_t *p) {

pthread_mutex_lock(&p->mutex);

alert_rule_t *rule = p->alert_rules;

while (rule) {

// 查找指标

time_series_t *ts = p->series;

while (ts) {

if (strcmp(ts->metric_name, rule->expr) == 0) {

if (ts->values) {

double value = ts->values->value;

int triggered = check_condition(value, rule->condition, rule->threshold);

if (triggered) {

// 检查是否已有告警

alert_event_t *evt = p->alert_events;

int exists = 0;

while (evt) {

if (strcmp(evt->rule_name, rule->name) == 0 &&

strcmp(evt->status, "firing") == 0) {

exists = 1;

break;

}

evt = evt->next;

}

if (!exists) {

evt = malloc(sizeof(alert_event_t));

strcpy(evt->rule_name, rule->name);

strcpy(evt->status, "firing");

evt->start_time = time(NULL);

evt->end_time = 0;

sprintf(evt->value, "%.2f", value);

evt->next = p->alert_events;

p->alert_events = evt;

printf(" $告警$ %s: %s (当前值: %.2f)\n",

rule->severity, rule->name, value);

}

} else {

// 关闭告警

alert_event_t *evt = p->alert_events;

while (evt) {

if (strcmp(evt->rule_name, rule->name) == 0 &&

strcmp(evt->status, "firing") == 0) {

strcpy(evt->status, "resolved");

evt->end_time = time(NULL);

printf(" $告警$ %s 已恢复\n", rule->name);

}

evt = evt->next;

}

break;

}

ts = ts->next;

}

rule = rule->next;

}

pthread_mutex_unlock(&p->mutex);

}

```

查询语言（PromQL简化）

```c

// 查询指标

metric_value_t *prom_query(prometheus_t *p, const char *metric_name,

label_t *labels, int label_count) {

pthread_mutex_lock(&p->mutex);

time_series_t *ts = p->series;

while (ts) {

if (strcmp(ts->metric_name, metric_name) != 0) {

ts = ts->next;

continue;

}

if (ts->label_count != label_count) {

ts = ts->next;

continue;

}

int match = 1;

for (int i = 0; i < label_count; i++) {

if (strcmp(ts->labels $i$ .key, labels $i$ .key) != 0 ||

strcmp(ts->labels $i$ .value, labels $i$ .value) != 0) {

match = 0;

break;

}

if (match && ts->values) {

pthread_mutex_unlock(&p->mutex);

return ts->values;

}

ts = ts->next;

}

pthread_mutex_unlock(&p->mutex);

return NULL;

}

// 聚合查询（rate/sum/avg）

double prom_query_rate(prometheus_t *p, const char *metric_name,

label_t *labels, int label_count) {

metric_value_t *current = prom_query(p, metric_name, labels, label_count);

if (!current || !current->next) return 0;

metric_value_t *prev = current->next;

double rate = (current->value - prev->value) /

(current->timestamp - prev->timestamp);

return rate * 1000; // 每秒速率

}

```

测试代码

```c

void test_prometheus() {

printf("=== Prometheus监控测试 ===\n\n");

prometheus_t *p = prom_create(5000, 7);

// 添加告警规则

prom_add_alert_rule(p, "CPU过高", "cpu_usage", ">", 80, 30,

"critical", "CPU使用率超过80%%");

prom_add_alert_rule(p, "内存不足", "memory_usage_mb", ">", 3000, 60,

"warning", "内存使用超过3GB");

// 模拟抓取10次

for (int i = 0; i < 10; i++) {

prom_scrape(p, "user-service");

prom_scrape(p, "order-service");

prom_scrape(p, "payment-service");

prom_evaluate_alerts(p);

printf("---\n");

usleep(100000);

}

// 查询测试

printf("\n=== 查询测试 ===\n");

label_t labels $2$ ;

strcpy(labels $0$ .key, "service");

strcpy(labels $0$ .value, "user-service");

strcpy(labels $1$ .key, "env");

strcpy(labels $1$ .value, "prod");

metric_value_t *cpu = prom_query(p, "cpu_usage", labels, 2);

if (cpu) {

printf("user-service CPU: %.2f%%\n", cpu->value);

}

printf("\n活跃告警:\n");

alert_event_t *evt = p->alert_events;

while (evt) {

if (strcmp(evt->status, "firing") == 0) {

printf(" %s: %s (值: %s)\n", evt->rule_name, evt->status, evt->value);

}

evt = evt->next;

}

free(p);

}

int main() {

srand(time(NULL));

test_prometheus();

return 0;

}

```

三、编译和运行

```bash

gcc -o prometheus prometheus.c -lpthread -lm

./prometheus

```

四、Prometheus vs 本实现

特性本实现 Prometheus

指标采集 ✅ Pull ✅ Pull

时间序列 ✅ ✅ TSDB

告警规则 ✅ ✅ Alertmanager

查询语言 ✅ 简化 ✅ PromQL

服务发现 ❌ ✅

可视化 ❌ ✅ Grafana

五、总结

通过这篇文章，你学会了：

· 监控系统核心原理（Pull模型）

· 指标类型（Counter/Gauge/Histogram）

· 时间序列存储

· 告警规则引擎

· 查询语言基础

Prometheus是云原生监控的事实标准。掌握它，你就理解了监控系统的核心设计。

下一篇预告：《从零实现一个分布式追踪：Jaeger的核心设计》

评论区分享一下你用Prometheus监控过什么系统～