【云原生】用Prometheus Operator监听Golang服务指标，并用Granafa可视化展示

一、任务

做一个golang服务，暴露/metrics接口，计算四个指标（Counter，Gauge，Histogram，Summary），Prometheus抓取这四个指标，并在Grafana中展示。

二、直接开始写代码

1.Golang代码

1.注册四个指标到prometheus
2.通过一个后台请求（一直存在）模拟三个请求到三个不同路由
3.代码运行在8086端口

package main

import (
	"math/rand"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/shirou/gopsutil/cpu"
)

var (
	requestCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "myapp_requests_total",
			Help: "Total number of requests",
		},
		[]string{"path"},
	)

	cpuUsage = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "myapp_cpu_usage",
			Help: "Current CPU usage",
		},
	)

	requestDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "myapp_request_duration_seconds",
			Help:    "Request duration in seconds",
			Buckets: prometheus.LinearBuckets(0.1, 0.1, 10),
		},
	)

	responseTime = prometheus.NewSummary(
		prometheus.SummaryOpts{
			Name:       "myapp_response_time_seconds",
			Help:       "Response time in seconds",
			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		},
	)
)

func init() {
	prometheus.MustRegister(requestCount, cpuUsage, requestDuration, responseTime)
}

func simulateRequests() {
	for {
		paths := []string{"/home", "/about", "/contact"}
		path := paths[rand.Intn(len(paths))]
		duration := 0.1 + 0.4*rand.Float64() // 模拟耗时 0.1~0.5 秒

		requestCount.WithLabelValues(path).Inc()
		requestDuration.Observe(duration)
		responseTime.Observe(duration)

		time.Sleep(time.Duration(rand.Intn(5)+1) * time.Second)
	}
}

func recordCPU() {
	for {
		percent, _ := cpu.Percent(time.Second, false)
		if len(percent) > 0 {
			cpuUsage.Set(percent[0])
		}
		time.Sleep(5 * time.Second)
	}
}

func main() {
	rand.Seed(time.Now().UnixNano())

	go simulateRequests()
	go recordCPU()

	http.Handle("/metrics", promhttp.Handler()) // 暴露指标
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello, this is a metrics demo application! Visit /metrics to see metrics."))
	})

	http.ListenAndServe(":8086", nil)
}

2.将Golang服务打包成镜像

FROM golang:1.23-alpine AS builder

# Set GOPROXY to use a more reliable proxy
ENV GOPROXY=https://goproxy.cn,direct

WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download

COPY . .
RUN go build -o metrics-demo .

FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /root/
COPY --from=builder /app/metrics-demo .
EXPOSE 8086
CMD ["./metrics-demo"]

3.写deployment 将Golang服务部署到pod。 写service，可通过8086端口访问服务，暴露/metrics

apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-demo
  namespace: monitoring-demo
  labels:
    app: metrics-demo
spec:
  replicas: 1
  selector:
    matchLabels:
      app: metrics-demo
  template:
    metadata:
      labels:
        app: metrics-demo
    spec:
      containers:
      - name: metrics-demo
        image: metrics-demo:latest
        ports:
        - containerPort: 8086
        resources:
          requests:
            memory: "64Mi"
            cpu: "250m"
          limits:
            memory: "128Mi"
            cpu: "500m"
---
apiVersion: v1
kind: Service
metadata:
  name: metrics-demo
  namespace: monitoring-demo
  labels:
    app: metrics-demo
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "8086"
    prometheus.io/path: "/metrics"
spec:
  selector:
    app: metrics-demo
  ports:
  - port: 8086
    targetPort: 8086
  type: ClusterIP

4.写serviceMonitor 来监听service

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: metrics-demo
  namespace: monitoring-demo
  labels:
    app: metrics-demo
spec:
  selector:
    matchLabels:
      app: metrics-demo
  endpoints:
  - port: "8086"
    path: /metrics
    interval: 30s

5.写关于Prometheus配置的yaml

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: prometheus-demo
  namespace: monitoring-demo
spec:
  serviceAccountName: prometheus
  serviceMonitorSelector:
    matchLabels:
      app: metrics-demo
  resources:
    requests:
      memory: 400Mi
  enableAdminAPI: false
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: monitoring-demo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups: [""]
  resources:
  - nodes
  - nodes/metrics
  - services
  - endpoints
  - pods
  verbs: ["get", "list", "watch"]
- apiGroups:
  - extensions
  resources:
  - ingresses
  verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: monitoring-demo

6.写运行grafana的yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana
  namespace: monitoring-demo
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      containers:
      - name: grafana
        image: grafana/grafana:latest
        ports:
        - containerPort: 3000
        env:
        - name: GF_SECURITY_ADMIN_USER
          value: admin
        - name: GF_SECURITY_ADMIN_PASSWORD
          value: admin
        resources:
          requests:
            memory: "128Mi"
            cpu: "100m"
          limits:
            memory: "256Mi"
            cpu: "200m"
---
apiVersion: v1
kind: Service
metadata:
  name: grafana
  namespace: monitoring-demo
spec:
  selector:
    app: grafana
  ports:
  - port: 3000
    targetPort: 3000
  type: LoadBalancer

7.写grafana的配置文件

{
  "dashboard": {
    "id": null,
    "title": "Go Application Metrics",
    "timezone": "browser",
    "schemaVersion": 16,
    "version": 0,
    "refresh": "25s",
    "panels": [
      {
        "id": 1,
        "type": "graph",
        "title": "Request Count by Path",
        "gridPos": {
          "x": 0,
          "y": 0,
          "w": 12,
          "h": 8
        },
        "targets": [
          {
            "expr": "rate(myapp_requests_total[5m])",
            "legendFormat": "{{path}}",
            "refId": "A"
          }
        ],
        "xaxis": {
          "mode": "time"
        },
        "yaxes": [
          {
            "format": "short",
            "label": "Requests/sec"
          },
          {
            "format": "short"
          }
        ]
      },
      {
        "id": 2,
        "type": "gauge",
        "title": "Current CPU Usage",
        "gridPos": {
          "x": 12,
          "y": 0,
          "w": 12,
          "h": 8
        },
        "targets": [
          {
            "expr": "myapp_cpu_usage",
            "refId": "A"
          }
        ],
        "options": {
          "fieldOptions": {
            "defaults": {
              "unit": "percentunit"
            }
          }
        }
      },
      {
        "id": 3,
        "type": "heatmap",
        "title": "Request Duration Distribution",
        "gridPos": {
          "x": 0,
          "y": 8,
          "w": 12,
          "h": 8
        },
        "targets": [
          {
            "expr": "rate(myapp_request_duration_seconds_bucket[5m])",
            "format": "heatmap",
            "refId": "A"
          }
        ],
        "xAxis": {
          "show": true
        },
        "yAxis": {
          "format": "s",
          "decimals": 2
        },
        "color": {
          "mode": "opacity"
        }
      },
      {
        "id": 4,
        "type": "graph",
        "title": "Response Time Quantiles",
        "gridPos": {
          "x": 12,
          "y": 8,
          "w": 12,
          "h": 8
        },
        "targets": [
          {
            "expr": "myapp_response_time_seconds{quantile=\"0.5\"}",
            "legendFormat": "50th percentile",
            "refId": "A"
          },
          {
            "expr": "myapp_response_time_seconds{quantile=\"0.9\"}",
            "legendFormat": "90th percentile",
            "refId": "B"
          },
          {
            "expr": "myapp_response_time_seconds{quantile=\"0.99\"}",
            "legendFormat": "99th percentile",
            "refId": "C"
          }
        ],
        "xaxis": {
          "mode": "time"
        },
        "yaxes": [
          {
            "format": "s",
            "label": "Response Time"
          },
          {
            "format": "short"
          }
        ]
      }
    ],
    "templating": {
      "list": []
    },
    "annotations": {
      "list": []
    },
    "links": []
  }
}

8. 写docker-compose.yml 将通过docker服务放在同一网络，以便可以相互访问

version: '3.8'

services:
  metrics-demo:
    build: .
    ports:
      - "8087:8086"
    networks:
      - monitoring

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    networks:
      - monitoring
    depends_on:
      - metrics-demo

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
    networks:
      - monitoring
    depends_on:
      - prometheus

networks:
  monitoring:

三、 最后总结

Go Metrics Demo with Prometheus and Grafana

This project demonstrates how to expose application metrics from a Go application, collect them with Prometheus Operator, and visualize them with Grafana.

Components

1. Go Application : Exposes custom metrics on /metrics endpoint
2. Prometheus Operator: Collects metrics from the application
3. Grafana: Visualizes the collected metrics

Prerequisites

• Kubernetes cluster (minikube, kind, or cloud provider)
• kubectl configured
• Docker (for building the image)

Quick Start

1. Build the Docker image:

   docker build -t metrics-demo:latest .

2. If using minikube, load the image:

   minikube image load metrics-demo:latest

3. Apply the Kubernetes manifests:

   kubectl apply -f k8s/namespace.yaml
   kubectl apply -f k8s/app-deployment.yaml
   kubectl apply -f k8s/servicemonitor.yaml
   kubectl apply -f k8s/prometheus.yaml
   kubectl apply -f k8s/grafana.yaml

4. Access the services:

   • Application metrics: kubectl port-forward svc/metrics-demo -n monitoring-demo 8086:8086
   • Prometheus: kubectl port-forward svc/prometheus-operated -n monitoring-demo 9090:9090
   • Grafana: kubectl port-forward svc/grafana -n monitoring-demo 3000:3000

5. In Grafana (http://localhost:3000):

   • Login with admin/admin
   • Add Prometheus datasource: http://prometheus-operated:9090
   • Create dashboards for metrics:
     • myapp_requests_total
     • myapp_cpu_usage
     • myapp_request_duration_seconds

Custom Metrics

The application exposes the following custom metrics:

• myapp_requests_total: Counter for total requests by path
• myapp_cpu_usage: Gauge for current CPU usage
• myapp_request_duration_seconds: Histogram for request durations
• myapp_response_time_seconds: Summary for response times

Development

To modify the application:

1. Edit main.go
2. Rebuild the Docker image
3. Redeploy the application