第7讲：大模型后端服务化——Go 实现高并发 LLM Proxy / Router / Fallback

一、开篇：为什么需要 LLM Proxy？

2026年，大模型 API 调用已经成为基础设施级流量。直接让业务代码调用 LLM API 会遇到四个核心问题：

问题	后果	LLM Proxy 的解法
多模型分散管理​	每个服务自己配 API Key、Endpoint，改模型要改代码	统一接入层，业务只认 Proxy
成本失控​	不同模型价格差百倍，没人知道谁花了多少钱	配额管理、成本告警、自动降级
稳定性脆弱​	单模型挂了全线瘫痪，API 限流导致连锁故障	多模型 Fallback、重试、熔断
缺乏可观测性​	不知道每次请求用了多久、花了多少钱、返回了什么	全量日志、Tracing、Metrics

LLM Proxy = 统一接入 + 智能路由 + 故障容错 + 成本管控

---

二、整体架构

┌─────────────────────────────────────────────────────────────────┐
│                        客户端（业务服务）                          │
└──────────────────────────┬──────────────────────────────────────┘
                           │ HTTP/gRPC
                           ▼
┌─────────────────────────────────────────────────────────────────┐
│                    LLM Proxy Service                             │
│                                                                  │
│  ┌──────────┐  ┌──────────┐  ┌──────────┐  ┌──────────┐        │
│  │ 认证鉴权  │  │ 速率限制  │  │ 语义缓存  │  │ 请求日志  │        │
│  └──────────┘  └──────────┘  └──────────┘  └──────────┘        │
│                                                                  │
│  ┌──────────────────────────────────────────────────────────┐   │
│  │                    Router Engine                          │   │
│  │  ┌──────────┐  ┌──────────┐  ┌──────────┐               │   │
│  │  │ 策略路由  │  │ 成本路由  │  │ 延迟路由  │               │   │
│  │  └──────────┘  └──────────┘  └──────────┘               │   │
│  └──────────────────────────────────────────────────────────┘   │
│                                                                  │
│  ┌──────────────────────────────────────────────────────────┐   │
│  │                    Model Backends                         │   │
│  │  ┌──────────┐  ┌──────────┐  ┌──────────┐  ┌──────────┐│   │
│  │  │ DeepSeek │  │  GPT-4o  │  │  Claude  │  │ 本地模型  ││   │
│  │  └──────────┘  └──────────┘  └──────────┘  └──────────┘│   │
│  └──────────────────────────────────────────────────────────┘   │
│                                                                  │
│  ┌──────────────────────────────────────────────────────────┐   │
│  │                  Circuit Breaker                          │   │
│  │  正常 ◄──► 熔断 ◄──► 半开 ◄──► 恢复                       │   │
│  └──────────────────────────────────────────────────────────┘   │
└─────────────────────────────────────────────────────────────────┘
                           │
                           ▼
                   OpenAI / Anthropic / 自有模型

---

三、核心数据结构

// proxy/models.go
package proxy

import (
    "context"
    "time"
)

// Provider 模型提供商
type Provider string

const (
    ProviderDeepSeek Provider = "deepseek"
    ProviderOpenAI   Provider = "openai"
    ProviderAnthropic Provider = "anthropic"
    ProviderLocal    Provider = "local"
)

// ModelInfo 模型信息
type ModelInfo struct {
    Name         string            `json:"name"`
    Provider     Provider          `json:"provider"`
    Endpoint     string            `json:"endpoint"`
    Capabilities []string          `json:"capabilities"` // chat, completion, embedding, vision
    PricePer1KInput  float64       `json:"price_per_1k_input"`   // 美元
    PricePer1KOutput float64       `json:"price_per_1k_output"`
    MaxTokens    int               `json:"max_tokens"`
    RateLimit    RateLimitConfig   `json:"rate_limit"`
    HealthCheck  HealthCheckConfig `json:"health_check"`
}

// RateLimitConfig 速率限制
type RateLimitConfig struct {
    RequestsPerMinute int `json:"rpm"`
    TokensPerMinute   int `json:"tpm"`
    BurstSize         int `json:"burst_size"`
}

// HealthCheckConfig 健康检查
type HealthCheckConfig struct {
    Enabled      bool          `json:"enabled"`
    Interval     time.Duration `json:"interval"`
    Timeout      time.Duration `json:"timeout"`
    FailureThreshold int       `json:"failure_threshold"`
}

// ChatRequest 统一的聊天请求
type ChatRequest struct {
    Model       string            `json:"model"`
    Messages    []Message         `json:"messages"`
    Temperature float32           `json:"temperature,omitempty"`
    MaxTokens   int               `json:"max_tokens,omitempty"`
    Stream      bool              `json:"stream,omitempty"`
    Metadata    map[string]string `json:"metadata,omitempty"` // 业务透传字段
}

// Message 消息
type Message struct {
    Role    string `json:"role"`
    Content string `json:"content"`
}

// ChatResponse 统一的聊天响应
type ChatResponse struct {
    Model      string `json:"model"`
    Message    Message `json:"message"`
    Usage      Usage  `json:"usage"`
    LatencyMs  int64  `json:"latency_ms"`
    CostUSD    float64 `json:"cost_usd"`
    FromCache  bool   `json:"from_cache"`
}

// Usage token 用量
type Usage struct {
    PromptTokens     int `json:"prompt_tokens"`
    CompletionTokens int `json:"completion_tokens"`
    TotalTokens      int `json:"total_tokens"`
}

// Backend 模型后端接口
type Backend interface {
    Name() string
    Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
    ChatStream(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error)
    CheckHealth(ctx context.Context) error
    GetModelInfo() *ModelInfo
}

// RouterStrategy 路由策略
type RouterStrategy string

const (
    StrategyPriority RouterStrategy = "priority"   // 按优先级
    StrategyCost     RouterStrategy = "cost"       // 按成本
    StrategyLatency  RouterStrategy = "latency"    // 按延迟
    StrategyRandom   RouterStrategy = "random"     // 随机
)

---

四、Backend 实现：统一适配器

// proxy/backend.go
package proxy

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "time"
)

// OpenAICompatibleBackend 兼容 OpenAI API 的后端
type OpenAICompatibleBackend struct {
    info    *ModelInfo
    apiKey  string
    client  *http.Client
}

func NewOpenAICompatibleBackend(info *ModelInfo, apiKey string) *OpenAICompatibleBackend {
    return &OpenAICompatibleBackend{
        info:   info,
        apiKey: apiKey,
        client: &http.Client{
            Timeout: 120 * time.Second,
            Transport: &http.Transport{
                MaxIdleConns:        100,
                IdleConnTimeout:     90 * time.Second,
                DisableCompression:  false,
            },
        },
    }
}

func (b *OpenAICompatibleBackend) Name() string {
    return b.info.Name
}

func (b *OpenAICompatibleBackend) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
    start := time.Now()

    // 构建 OpenAI 兼容请求体
    body := map[string]interface{}{
        "model":    b.info.Name,
        "messages": req.Messages,
        "stream":   false,
    }
    if req.Temperature > 0 {
        body["temperature"] = req.Temperature
    }
    if req.MaxTokens > 0 {
        body["max_tokens"] = req.MaxTokens
    }

    jsonBody, _ := json.Marshal(body)
    httpReq, _ := http.NewRequestWithContext(ctx, "POST", b.info.Endpoint+"/chat/completions", bytes.NewReader(jsonBody))
    httpReq.Header.Set("Content-Type", "application/json")
    httpReq.Header.Set("Authorization", "Bearer "+b.apiKey)

    resp, err := b.client.Do(httpReq)
    if err != nil {
        return nil, fmt.Errorf("request failed: %w", err)
    }
    defer resp.Body.Close()

    respBody, _ := io.ReadAll(resp.Body)

    if resp.StatusCode != 200 {
        return nil, fmt.Errorf("API error [%d]: %s", resp.StatusCode, string(respBody))
    }

    // 解析 OpenAI 响应
    var openAIResp struct {
        Choices []struct {
            Message struct {
                Role    string `json:"role"`
                Content string `json:"content"`
            } `json:"message"`
        } `json:"choices"`
        Usage struct {
            PromptTokens     int `json:"prompt_tokens"`
            CompletionTokens int `json:"completion_tokens"`
            TotalTokens      int `json:"total_tokens"`
        } `json:"usage"`
    }
    json.Unmarshal(respBody, &openAIResp)

    latency := time.Since(start).Milliseconds()
    usage := openAIResp.Usage

    return &ChatResponse{
        Model: b.info.Name,
        Message: Message{
            Role:    openAIResp.Choices[0].Message.Role,
            Content: openAIResp.Choices[0].Message.Content,
        },
        Usage: Usage{
            PromptTokens:     usage.PromptTokens,
            CompletionTokens: usage.CompletionTokens,
            TotalTokens:      usage.TotalTokens,
        },
        LatencyMs: latency,
        CostUSD:   calculateCost(b.info, usage.PromptTokens, usage.CompletionTokens),
    }, nil
}

func (b *OpenAICompatibleBackend) ChatStream(ctx context.Context, req *ChatRequest) (<-chan *ChatResponse, error) {
    ch := make(chan *ChatResponse, 16)

    go func() {
        defer close(ch)

        body := map[string]interface{}{
            "model":    b.info.Name,
            "messages": req.Messages,
            "stream":   true,
        }
        jsonBody, _ := json.Marshal(body)

        httpReq, _ := http.NewRequestWithContext(ctx, "POST", b.info.Endpoint+"/chat/completions", bytes.NewReader(jsonBody))
        httpReq.Header.Set("Content-Type", "application/json")
        httpReq.Header.Set("Authorization", "Bearer "+b.apiKey)

        resp, err := b.client.Do(httpReq)
        if err != nil {
            ch <- &ChatResponse{Message: Message{Role: "error", Content: err.Error()}}
            return
        }
        defer resp.Body.Close()

        decoder := json.NewDecoder(resp.Body)
        for {
            var event struct {
                Choices []struct {
                    Delta struct {
                        Content string `json:"content"`
                    } `json:"delta"`
                    FinishReason string `json:"finish_reason"`
                } `json:"choices"`
            }
            if err := decoder.Decode(&event); err != nil {
                break
            }

            if len(event.Choices) > 0 {
                ch <- &ChatResponse{
                    Model: b.info.Name,
                    Message: Message{
                        Role:    "assistant",
                        Content: event.Choices[0].Delta.Content,
                    },
                }

                if event.Choices[0].FinishReason == "stop" {
                    break
                }
            }
        }
    }()

    return ch, nil
}

func (b *OpenAICompatibleBackend) CheckHealth(ctx context.Context) error {
    httpReq, _ := http.NewRequestWithContext(ctx, "GET", b.info.Endpoint+"/models", nil)
    httpReq.Header.Set("Authorization", "Bearer "+b.apiKey)
    
    resp, err := b.client.Do(httpReq)
    if err != nil {
        return err
    }
    defer resp.Body.Close()
    
    if resp.StatusCode != 200 {
        return fmt.Errorf("health check failed: %d", resp.StatusCode)
    }
    return nil
}

func (b *OpenAICompatibleBackend) GetModelInfo() *ModelInfo {
    return b.info
}

func calculateCost(info *ModelInfo, promptTokens, completionTokens int) float64 {
    cost := (float64(promptTokens) / 1000.0 * info.PricePer1KInput) +
        (float64(completionTokens) / 1000.0 * info.PricePer1KOutput)
    return cost
}

---

五、Router：智能路由引擎

// proxy/router.go
package proxy

import (
    "context"
    "math/rand"
    "sort"
    "sync"
    "time"
)

// Router 路由引擎
type Router struct {
    backends     map[string]Backend
    strategy     RouterStrategy
    priorities   map[string]int // backend name -> priority (越小越优先)
    latencyStats map[string]*LatencyStat
    mu           sync.RWMutex
}

type LatencyStat struct {
    AvgLatencyMs float64
    SampleCount  int64
    LastUpdated  time.Time
}

func NewRouter(strategy RouterStrategy) *Router {
    return &Router{
        backends:     make(map[string]Backend),
        strategy:     strategy,
        priorities:   make(map[string]int),
        latencyStats: make(map[string]*LatencyStat),
    }
}

// RegisterBackend 注册后端模型
func (r *Router) RegisterBackend(backend Backend, priority int) {
    r.mu.Lock()
    defer r.mu.Unlock()
    r.backends[backend.Name()] = backend
    r.priorities[backend.Name()] = priority
    r.latencyStats[backend.Name()] = &LatencyStat{}
}

// SelectBackend 选择最优后端
func (r *Router) SelectBackend(ctx context.Context, req *ChatRequest) (Backend, error) {
    r.mu.RLock()
    defer r.mu.RUnlock()

    if len(r.backends) == 0 {
        return nil, fmt.Errorf("no backends available")
    }

    // 如果指定了模型，直接匹配
    if req.Model != "" {
        if backend, ok := r.backends[req.Model]; ok {
            return backend, nil
        }
    }

    switch r.strategy {
    case StrategyPriority:
        return r.selectByPriority()
    case StrategyCost:
        return r.selectByCost(req)
    case StrategyLatency:
        return r.selectByLatency()
    case StrategyRandom:
        return r.selectRandom()
    default:
        return r.selectByPriority()
    }
}

func (r *Router) selectByPriority() (Backend, error) {
    type entry struct {
        name     string
        priority int
    }
    
    entries := make([]entry, 0, len(r.backends))
    for name, priority := range r.priorities {
        entries = append(entries, entry{name, priority})
    }
    
    sort.Slice(entries, func(i, j int) bool {
        return entries[i].priority < entries[j].priority
    })
    
    // 尝试按优先级依次选择，跳过不可用的
    for _, e := range entries {
        if backend, ok := r.backends[e.name]; ok {
            return backend, nil
        }
    }
    
    return nil, fmt.Errorf("no available backend")
}

func (r *Router) selectByCost(req *ChatRequest) (Backend, error) {
    // 估算 token 数量（简化版：按字符数估算）
    estimatedTokens := 0
    for _, msg := range req.Messages {
        estimatedTokens += len(msg.Content) / 2 // 粗略估计
    }
    
    type costEntry struct {
        name string
        cost float64
    }
    
    var candidates []costEntry
    for name, backend := range r.backends {
        info := backend.GetModelInfo()
        cost := (float64(estimatedTokens) / 1000.0 * info.PricePer1KInput) +
            (float64(req.MaxTokens) / 1000.0 * info.PricePer1KOutput)
        candidates = append(candidates, costEntry{name, cost})
    }
    
    sort.Slice(candidates, func(i, j int) bool {
        return candidates[i].cost < candidates[j].cost
    })
    
    if len(candidates) > 0 {
        return r.backends[candidates[0].name], nil
    }
    return nil, fmt.Errorf("no backend available")
}

func (r *Router) selectByLatency() (Backend, error) {
    type latencyEntry struct {
        name    string
        latency float64
    }
    
    var candidates []latencyEntry
    for name, stat := range r.latencyStats {
        if stat.SampleCount > 0 {
            candidates = append(candidates, latencyEntry{name, stat.AvgLatencyMs})
        }
    }
    
    if len(candidates) == 0 {
        return r.selectRandom()
    }
    
    sort.Slice(candidates, func(i, j int) bool {
        return candidates[i].latency < candidates[j].latency
    })
    
    return r.backends[candidates[0].name], nil
}

func (r *Router) selectRandom() (Backend, error) {
    names := make([]string, 0, len(r.backends))
    for name := range r.backends {
        names = append(names, name)
    }
    return r.backends[names[rand.Intn(len(names))]], nil
}

// RecordLatency 记录延迟，用于后续路由决策
func (r *Router) RecordLatency(backendName string, latencyMs int64) {
    r.mu.Lock()
    defer r.mu.Unlock()
    
    stat, ok := r.latencyStats[backendName]
    if !ok {
        return
    }
    
    // 指数移动平均
    alpha := 0.1
    if stat.SampleCount == 0 {
        stat.AvgLatencyMs = float64(latencyMs)
    } else {
        stat.AvgLatencyMs = (1-alpha)*stat.AvgLatencyMs + alpha*float64(latencyMs)
    }
    stat.SampleCount++
    stat.LastUpdated = time.Now()
}

---

六、Circuit Breaker：熔断器

// proxy/circuitbreaker.go
package proxy

import (
    "context"
    "errors"
    "sync"
    "sync/atomic"
    "time"
)

// State 熔断器状态
type State int32

const (
    StateClosed   State = 0 // 正常
    StateOpen     State = 1 // 熔断开启
    StateHalfOpen State = 2 // 半开
)

// CircuitBreaker 熔断器
type CircuitBreaker struct {
    name             string
    state            atomic.Value // State
    failureCount     int64
    successCount     int64
    halfOpenSuccess  int64
    
    threshold        int64         // 触发熔断的失败次数
    recoveryTimeout  time.Duration // 熔断后多久进入半开
    halfOpenMaxCalls int64         // 半开状态下允许的最大请求数
    
    lastFailureTime  time.Time
    mu               sync.Mutex
}

func NewCircuitBreaker(name string, threshold int64, recoveryTimeout time.Duration, halfOpenMaxCalls int64) *CircuitBreaker {
    cb := &CircuitBreaker{
        name:             name,
        threshold:        threshold,
        recoveryTimeout:  recoveryTimeout,
        halfOpenMaxCalls: halfOpenMaxCalls,
    }
    cb.state.Store(StateClosed)
    return cb
}

// Execute 执行请求，受熔断器保护
func (cb *CircuitBreaker) Execute(ctx context.Context, fn func(context.Context) (interface{}, error)) (interface{}, error) {
    if !cb.allowRequest() {
        return nil, errors.New("circuit breaker is open")
    }

    result, err := fn(ctx)
    cb.recordResult(err == nil)
    return result, err
}

func (cb *CircuitBreaker) allowRequest() bool {
    state := cb.state.Load().(State)
    
    switch state {
    case StateClosed:
        return true
    case StateOpen:
        // 检查是否到了恢复时间
        cb.mu.Lock()
        defer cb.mu.Unlock()
        if time.Since(cb.lastFailureTime) > cb.recoveryTimeout {
            cb.state.Store(StateHalfOpen)
            atomic.StoreInt64(&cb.halfOpenSuccess, 0)
            return true
        }
        return false
    case StateHalfOpen:
        // 半开状态允许一定数量的请求通过
        success := atomic.LoadInt64(&cb.halfOpenSuccess)
        return success < cb.halfOpenMaxCalls
    default:
        return false
    }
}

func (cb *CircuitBreaker) recordResult(success bool) {
    if success {
        atomic.AddInt64(&cb.successCount, 1)
        
        state := cb.state.Load().(State)
        if state == StateHalfOpen {
            atomic.AddInt64(&cb.halfOpenSuccess, 1)
            
            // 半开状态下连续成功达到阈值，关闭熔断器
            if atomic.LoadInt64(&cb.halfOpenSuccess) >= cb.halfOpenMaxCalls {
                cb.mu.Lock()
                cb.state.Store(StateClosed)
                atomic.StoreInt64(&cb.failureCount, 0)
                cb.mu.Unlock()
            }
        }
    } else {
        atomic.AddInt64(&cb.failureCount, 1)
        cb.mu.Lock()
        cb.lastFailureTime = time.Now()
        cb.mu.Unlock()
        
        failures := atomic.LoadInt64(&cb.failureCount)
        if failures >= cb.threshold {
            cb.mu.Lock()
            cb.state.Store(StateOpen)
            cb.mu.Unlock()
        }
    }
}

func (cb *CircuitBreaker) State() State {
    return cb.state.Load().(State)
}

func (cb *CircuitBreaker) Reset() {
    cb.mu.Lock()
    defer cb.mu.Unlock()
    cb.state.Store(StateClosed)
    atomic.StoreInt64(&cb.failureCount, 0)
    atomic.StoreInt64(&cb.successCount, 0)
    atomic.StoreInt64(&cb.halfOpenSuccess, 0)
}

---

七、Semantic Cache：语义缓存

// proxy/cache.go
package proxy

import (
    "context"
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "fmt"
    "sync"
    "time"
    
    "github.com/google/uuid"
)

// CacheEntry 缓存条目
type CacheEntry struct {
    Response   *ChatResponse
    CreatedAt  time.Time
    ExpiresAt  time.Time
    HitCount   int64
    Embedding  []float32 // 语义嵌入
}

// SemanticCache 语义缓存
type SemanticCache struct {
    exactCache map[string]*CacheEntry    // 精确匹配缓存
    semanticDB []*CacheEntry             // 语义缓存（简化版用切片）
    mu         sync.RWMutex
    ttl        time.Duration
    similarity float64                   // 语义相似度阈值
    maxEntries int
}

func NewSemanticCache(ttl time.Duration, similarity float64, maxEntries int) *SemanticCache {
    return &SemanticCache{
        exactCache: make(map[string]*CacheEntry),
        semanticDB: make([]*CacheEntry, 0, maxEntries),
        ttl:        ttl,
        similarity: similarity,
        maxEntries: maxEntries,
    }
}

// exactKey 生成精确匹配的 key
func exactKey(req *ChatRequest) string {
    data, _ := json.Marshal(req)
    hash := sha256.Sum256(data)
    return hex.EncodeToString(hash[:])
}

// Get 获取缓存
func (c *SemanticCache) Get(ctx context.Context, req *ChatRequest) (*ChatResponse, bool) {
    c.mu.RLock()
    defer c.mu.RUnlock()

    // 1. 先查精确匹配
    key := exactKey(req)
    if entry, ok := c.exactCache[key]; ok {
        if time.Now().Before(entry.ExpiresAt) {
            entry.HitCount++
            return entry.Response, true
        }
        delete(c.exactCache, key)
    }

    // 2. 语义匹配（简化版：遍历所有条目）
    // 生产环境应该用向量数据库
    for _, entry := range c.semanticDB {
        if time.Now().After(entry.ExpiresAt) {
            continue
        }
        
        // 计算语义相似度（简化版：比较消息内容的前缀）
        similarity := computeSimilarity(req, entry.Response)
        if similarity >= c.similarity {
            entry.HitCount++
            return entry.Response, true
        }
    }

    return nil, false
}

// Set 设置缓存
func (c *SemanticCache) Set(ctx context.Context, req *ChatRequest, resp *ChatResponse) {
    c.mu.Lock()
    defer c.mu.Unlock()

    entry := &CacheEntry{
        Response:  resp,
        CreatedAt: time.Now(),
        ExpiresAt: time.Now().Add(c.ttl),
    }

    // 精确缓存
    key := exactKey(req)
    c.exactCache[key] = entry

    // 语义缓存
    c.semanticDB = append(c.semanticDB, entry)
    if len(c.semanticDB) > c.maxEntries {
        // 淘汰最旧的
        c.semanticDB = c.semanticDB[len(c.semanticDB)-c.maxEntries:]
    }
}

// computeSimilarity 计算相似度（简化版）
func computeSimilarity(req *ChatRequest, cachedResp *ChatResponse) float64 {
    // 生产环境应该用 embedding 向量余弦相似度
    // 这里简化为：如果请求的最后一条消息包含缓存的响应的前20个字符，认为相似
    if len(req.Messages) == 0 || cachedResp == nil {
        return 0
    }
    
    lastMsg := req.Messages[len(req.Messages)-1].Content
    cachedContent := cachedResp.Message.Content
    
    if len(lastMsg) > 50 && len(cachedContent) > 20 {
        prefix := cachedContent[:20]
        if len(lastMsg) >= len(prefix) && lastMsg[:len(prefix)] == prefix {
            return 0.95
        }
    }
    
    return 0
}

// Stats 缓存统计
func (c *SemanticCache) Stats() map[string]interface{} {
    c.mu.RLock()
    defer c.mu.RUnlock()
    
    totalHits := int64(0)
    for _, entry := range c.exactCache {
        totalHits += entry.HitCount
    }
    for _, entry := range c.semanticDB {
        totalHits += entry.HitCount
    }
    
    return map[string]interface{}{
        "exact_entries": len(c.exactCache),
        "semantic_entries": len(c.semanticDB),
        "total_hits": totalHits,
    }
}

---

八、Rate Limiter：速率限制

// proxy/ratelimit.go
package proxy

import (
    "sync"
    "time"
)

// TokenBucket 令牌桶
type TokenBucket struct {
    rate       float64   // 每秒填充速率
    capacity   float64   // 桶容量
    tokens     float64   // 当前令牌数
    lastRefill time.Time
    mu         sync.Mutex
}

func NewTokenBucket(rate float64, capacity float64) *TokenBucket {
    return &TokenBucket{
        rate:       rate,
        capacity:   capacity,
        tokens:     capacity,
        lastRefill: time.Now(),
    }
}

func (tb *TokenBucket) Allow() bool {
    tb.mu.Lock()
    defer tb.mu.Unlock()

    now := time.Now()
    elapsed := now.Sub(tb.lastRefill).Seconds()
    tb.tokens = min(tb.capacity, tb.tokens+elapsed*tb.rate)
    tb.lastRefill = now

    if tb.tokens >= 1 {
        tb.tokens--
        return true
    }
    return false
}

func min(a, b float64) float64 {
    if a < b {
        return a
    }
    return b
}

// MultiRateLimiter 多维度限流
type MultiRateLimiter struct {
    globalRPM  *TokenBucket // 全局每分钟请求数
    globalTPM  *TokenBucket // 全局每分钟 token 数
    perUser    map[string]*TokenBucket
    perIP      map[string]*TokenBucket
    mu         sync.RWMutex
}

func NewMultiRateLimiter(rpm, tpm int) *MultiRateLimiter {
    return &MultiRateLimiter{
        globalRPM: NewTokenBucket(float64(rpm)/60.0, float64(rpm)),
        globalTPM: NewTokenBucket(float64(tpm)/60.0, float64(tpm)),
        perUser:   make(map[string]*TokenBucket),
        perIP:     make(map[string]*TokenBucket),
    }
}

func (rl *MultiRateLimiter) Allow(userID, ip string, estimatedTokens int) bool {
    // 全局 RPM 检查
    if !rl.globalRPM.Allow() {
        return false
    }

    // 全局 TPM 检查（简化版：每次消耗 estimatedTokens 个令牌）
    if !rl.globalTPM.Allow() {
        return false
    }

    // 用户级别限流（每个用户 100 RPM）
    rl.mu.Lock()
    userBucket, ok := rl.perUser[userID]
    if !ok {
        userBucket = NewTokenBucket(100.0/60.0, 100.0)
        rl.perUser[userID] = userBucket
    }
    rl.mu.Unlock()

    if !userBucket.Allow() {
        return false
    }

    return true
}

---

九、Proxy Server：完整服务

// proxy/server.go
package proxy

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "net/http"
    "time"
)

// ProxyServer LLM Proxy 服务
type ProxyServer struct {
    router    *Router
    cache     *SemanticCache
    limiter   *MultiRateLimiter
    breakers  map[string]*CircuitBreaker
    backends  []Backend
}

func NewProxyServer(router *Router, cache *SemanticCache, limiter *MultiRateLimiter) *ProxyServer {
    return &ProxyServer{
        router:   router,
        cache:    cache,
        limiter:  limiter,
        breakers: make(map[string]*CircuitBreaker),
    }
}

// AddBackend 添加后端并注册熔断器
func (s *ProxyServer) AddBackend(backend Backend, priority int) {
    s.backends = append(s.backends, backend)
    s.router.RegisterBackend(backend, priority)
    
    s.breakers[backend.Name()] = NewCircuitBreaker(
        backend.Name(),
        5,                    // 连续5次失败触发熔断
        30*time.Second,       // 30秒后尝试恢复
        3,                    // 半开状态允许3个请求
    )
}

// HandleChat 处理聊天请求
func (s *ProxyServer) HandleChat(w http.ResponseWriter, r *http.Request) {
    if r.Method != "POST" {
        http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
        return
    }

    var req ChatRequest
    if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
        http.Error(w, fmt.Sprintf("Invalid request: %v", err), http.StatusBadRequest)
        return
    }

    ctx := r.Context()

    // 1. 限流检查
    userID := r.Header.Get("X-User-ID")
    if userID == "" {
        userID = "anonymous"
    }
    ip := r.RemoteAddr

    estimatedTokens := estimateTokens(&req)
    if !s.limiter.Allow(userID, ip, estimatedTokens) {
        http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
        return
    }

    // 2. 缓存检查
    if !req.Stream {
        if cached, ok := s.cache.Get(ctx, &req); ok {
            w.Header().Set("Content-Type", "application/json")
            cached.FromCache = true
            json.NewEncoder(w).Encode(cached)
            return
        }
    }

    // 3. 选择后端
    backend, err := s.router.SelectBackend(ctx, &req)
    if err != nil {
        http.Error(w, fmt.Sprintf("No available backend: %v", err), http.StatusServiceUnavailable)
        return
    }

    // 4. 通过熔断器执行
    breaker := s.breakers[backend.Name()]
    if breaker == nil {
        http.Error(w, "Backend not configured", http.StatusInternalServerError)
        return
    }

    result, err := breaker.Execute(ctx, func(ctx context.Context) (interface{}, error) {
        if req.Stream {
            return s.handleStream(ctx, backend, &req, w)
        }
        return backend.Chat(ctx, &req)
    })

    if err != nil {
        // 5. Fallback：尝试下一个可用后端
        fallbackResp := s.tryFallback(ctx, &req, backend.Name())
        if fallbackResp != nil {
            w.Header().Set("Content-Type", "application/json")
            json.NewEncoder(w).Encode(fallbackResp)
            return
        }

        http.Error(w, fmt.Sprintf("All backends failed: %v", err), http.StatusServiceUnavailable)
        return
    }

    // 6. 写入缓存
    if resp, ok := result.(*ChatResponse); ok && !req.Stream {
        s.cache.Set(ctx, &req, resp)
        s.router.RecordLatency(backend.Name(), resp.LatencyMs)
    }

    // 7. 记录日志
    s.logRequest(&req, result, backend.Name())

    w.Header().Set("Content-Type", "application/json")
    json.NewEncoder(w).Encode(result)
}

func (s *ProxyServer) handleStream(ctx context.Context, backend Backend, req *ChatRequest, w http.ResponseWriter) (interface{}, error) {
    flusher, ok := w.(http.Flusher)
    if !ok {
        return nil, fmt.Errorf("streaming not supported")
    }

    w.Header().Set("Content-Type", "text/event-stream")
    w.Header().Set("Cache-Control", "no-cache")
    w.Header().Set("Connection", "keep-alive")

    stream, err := backend.ChatStream(ctx, req)
    if err != nil {
        return nil, err
    }

    for chunk := range stream {
        data, _ := json.Marshal(chunk)
        fmt.Fprintf(w, "data: %s\n\n", data)
        flusher.Flush()
    }

    fmt.Fprint(w, "data: [DONE]\n\n")
    flusher.Flush()

    return nil, nil
}

func (s *ProxyServer) tryFallback(ctx context.Context, req *ChatRequest, excludeName string) *ChatResponse {
    // 尝试其他后端
    for _, backend := range s.backends {
        if backend.Name() == excludeName {
            continue
        }

        breaker := s.breakers[backend.Name()]
        if breaker.State() == StateOpen {
            continue
        }

        result, err := backend.Chat(ctx, req)
        if err == nil {
            log.Printf("Fallback to %s succeeded", backend.Name())
            return result
        }
    }

    return nil
}

func (s *ProxyServer) logRequest(req *ChatRequest, result interface{}, backendName string) {
    // 生产环境：写入 ELK / Loki / Prometheus
    log.Printf("[LLM Proxy] model=%s backend=%s user=%s tokens=%d",
        req.Model, backendName, req.Metadata["user_id"], 
        func() int {
            if resp, ok := result.(*ChatResponse); ok {
                return resp.Usage.TotalTokens
            }
            return 0
        }())
}

func estimateTokens(req *ChatRequest) int {
    tokens := 0
    for _, msg := range req.Messages {
        tokens += len(msg.Content) / 2
    }
    return max(tokens, 100)
}

func max(a, b int) int {
    if a > b {
        return a
    }
    return b
}

// Start 启动 HTTP 服务
func (s *ProxyServer) Start(addr string) error {
    mux := http.NewServeMux()
    mux.HandleFunc("/v1/chat/completions", s.HandleChat)
    mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
        w.WriteHeader(http.StatusOK)
        json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
    })

    log.Printf("LLM Proxy starting on %s", addr)
    return http.ListenAndServe(addr, mux)
}

---

十、启动示例

// cmd/main.go
package main

import (
    "time"
    "proxy"
)

func main() {
    // 1. 创建路由引擎（按优先级路由）
    router := proxy.NewRouter(proxy.StrategyPriority)

    // 2. 创建语义缓存（TTL=5分钟，相似度阈值0.85，最多缓存10000条）
    cache := proxy.NewSemanticCache(5*time.Minute, 0.85, 10000)

    // 3. 创建限流器（全局 1000 RPM, 500000 TPM）
    limiter := proxy.NewMultiRateLimiter(1000, 500000)

    // 4. 创建 Proxy Server
    server := proxy.NewProxyServer(router, cache, limiter)

    // 5. 注册后端模型
    server.AddBackend(proxy.NewOpenAICompatibleBackend(&proxy.ModelInfo{
        Name:            "deepseek-chat",
        Provider:        proxy.ProviderDeepSeek,
        Endpoint:        "https://api.deepseek.com",
        PricePer1KInput:  0.014,
        PricePer1KOutput: 0.028,
        MaxTokens:       8192,
    }, "sk-deepseek-key"), 10) // 优先级 10（最高）

    server.AddBackend(proxy.NewOpenAICompatibleBackend(&proxy.ModelInfo{
        Name:            "gpt-4o-mini",
        Provider:        proxy.ProviderOpenAI,
        Endpoint:        "https://api.openai.com",
        PricePer1KInput:  0.015,
        PricePer1KOutput: 0.060,
        MaxTokens:       16384,
    }, "sk-openai-key"), 20) // 优先级 20

    server.AddBackend(proxy.NewOpenAICompatibleBackend(&proxy.ModelInfo{
        Name:            "claude-3-haiku",
        Provider:        proxy.ProviderAnthropic,
        Endpoint:        "https://api.anthropic.com",
        PricePer1KInput:  0.025,
        PricePer1KOutput: 0.125,
        MaxTokens:       8192,
    }, "sk-anthropic-key"), 30) // 优先级 30

    // 6. 启动服务
    server.Start(":8080")
}

---

十一、关键要点

1. 统一接入层是第一步 --- 所有业务通过 Proxy 调用 LLM，而不是直连 API
2. 路由策略决定成本和体验 --- 优先级路由保稳定，成本路由省钱，延迟路由保速度
3. 熔断器防止雪崩 --- 一个后端挂了不会拖垮整个系统
4. 语义缓存节省 30%~50% 成本 --- 相似问题的回复可以复用
5. 多维度限流保护后端 --- 全局、用户、IP 三级限流
6. Fallback 链保证高可用 --- 主模型挂了自动切备用，用户无感知
7. 全量可观测性是标配 --- 每次请求都要记录模型、耗时、成本、token 数

---

十二、预告

第 8 讲预告：《RAG 生产化：Go 实现亿级向量召回 + 混合检索 + 重排序 Pipeline》

你将学到：

• 亿级向量的存储和检索方案（Milvus/Qdrant 生产配置）
• 混合检索：向量召回 + BM25 关键词 + 结构化过滤
• 重排序（Reranker）：Cross-Encoder 精排，提升 Top-K 准确率
• 完整的 RAG Pipeline：Query Rewrite → Retrieval → Rerank → Generation
• Go 实现的生产级 RAG 中间件

---

🧰 开发之余的小工具推荐

处理 Base64、JSON 格式化、JWT 解析、Crontab 计算、PDF 合并压缩这些碎片需求，我常用一个纯前端本地工具箱：zz365.top。所有计算在浏览器完成，文件不上服务器，关页即清。免费、无登录、无广告，适合开发者当常驻标签页。