k8s集群部署vector日志采集器

helm部署vector日志采集器

部署

shell 复制代码

# 集群内部署，采集容器输出到控制台的日志
helm install vector vector/vector --version 0.45.0 -n vector -f values.yaml
helm upgrade vector vector/vector --version 0.45.0 -n vector -f values.yaml

配置values文件

yaml 复制代码

role: Agent

# 添加一个启动引导容器，拉取GeoLite2-City.mmdb数据库，用于后期日志分析
initContainers:
  - name: geolite-database-loader
    image: busybox:latest
    imagePullPolicy: IfNotPresent
    command:
      [
        "sh",
        "-c",
        "wget -O /vector-data-dir/GeoLite2-City.mmdb https://git.io/GeoLite2-City.mmdb",
      ]
    volumeMounts:
      - name: data
        mountPath: /vector-data-dir

service:
  enabled: true
  type: "ClusterIP"
  ports:
    - port: 9598
      targetPort: 9598
      name: prom-exporter
      protocol: TCP

containerPorts:
  - containerPort: 9598
    name: prom-exporter
    protocol: TCP

customConfig:

  enrichment_tables:
    geoip_table:
      path: /vector-data-dir/GeoLite2-City.mmdb
      type: geoip
  data_dir: /vector-data-dir
  
  sources:
  # 按照命名空间采集不同的日志流
    kubernetes_logs_ns_prod:
      type: kubernetes_logs
      extra_field_selector: metadata.namespace=prod
    kubernetes_logs_ns_higress:
      type: kubernetes_logs
      extra_field_selector: metadata.namespace=higress-system
  # 日志转换
  transforms:
    kubernetes_container_logs_ns_prod:
      type: remap
      inputs:
        - kubernetes_logs_ns_prod
      source: |
        .message = .message
        .container = .kubernetes.container_name
        .pod = .kubernetes.pod_name
        .namespace = .kubernetes.pod_namespace
        del(.kubernetes)
        del(.file)
        del(.source_type)
        del(.stream)
        del(._timestamp)

    kubernetes_container_logs_ns_higress:
      type: remap
      inputs:
        - kubernetes_logs_ns_higress
      source: |
        .container = .kubernetes.container_name
        .pod = .kubernetes.pod_name
        .namespace = .kubernetes.pod_namespace

        # 解析 message 中的 JSON
        parsed, err = parse_json(.message)
        if err == null && is_object(parsed) {
          . = merge!(., parsed)
          del(.message)
        }

        # ---------------- UA 解析开始 ----------------
        # 组装 ua_str
        ua_str = null
        if exists(.user_agent) && is_string(.user_agent) {
          ua_str = .user_agent
        } else if exists(.ua) && is_string(.ua) {
          ua_str = .ua
        } else if exists(.headers) && is_object(.headers) {
          if exists(.headers."user-agent") && is_string(.headers."user-agent") {
            ua_str = .headers."user-agent"
          } else if exists(.headers."User-Agent") && is_string(.headers."User-Agent") {
            ua_str = .headers."User-Agent"
          }
        }

        # 强制版本调用前先做类型检查，避免 E103
        if ua_str != null && is_string(ua_str) {
          ua = parse_user_agent!(ua_str)
          if ua != null {
            .ua = ua
            .ua_browser = ua.browser
            .ua_os = ua.os
            .ua_device = ua.device
          }
        }
        # ---------------- UA 解析结束 ----------------

        # ---------------- IP 地理位置解析（从 x_forwarded_for） ----------------
        if exists(.x_forwarded_for) && is_string(.x_forwarded_for) {
          parts, perr = split(.x_forwarded_for, ",")
          if perr == null && is_array(parts) && length(parts) > 0 {
            ip_raw = parts[0]
            if is_string(ip_raw) {
              # 先定义 ip，确保变量存在
              ip = replace!(ip_raw, r'^\s+|\s+$', "")
              # 仅在 ip 已是字符串后再做长度判断，避免 E110
              if is_string(ip) && length(ip) > 0 {
                # enrichment table 查询用可失败赋值，避免 E103
                geo, gerr = get_enrichment_table_record("geoip_table", {"ip": ip}, ["country_code", "country_name", "region_name", "city_name","latitude","longitude"])
                if gerr == null && geo != null {
                  # 不同版本可能返回数组或对象，做兼容
                  if is_array(geo) && length(geo) > 0 {
                    .geoip = geo[0]
                  } else if is_object(geo) {
                    .geoip = geo
                  }
                }
              }
            }
          }
        }
        # ---------------- IP 地理位置解析结束 ----------------

        del(.kubernetes)
        del(.file)
        del(.source_type)
        del(.stream)
        del(._timestamp)
  # 输出日志到日志系统，以openobserve为例
  sinks:
    openobserve_ns_prod:
      type: http
      uri: http://123.123.123.123:5080/api/default/ns_prod_logs/_json
      compression: gzip
      encoding:
        codec: json
      inputs:
        - kubernetes_container_logs_ns_prod
      auth:
        strategy: basic
        user: test@openobserve.com
        password: qweasd

    openobserve_ns_hiress:
      type: http
      uri: http://123.123.123.123:5080/api/default/ns_higress_logs/_json
      compression: gzip
      encoding:
        codec: json
      inputs:
        - kubernetes_container_logs_ns_higress
      auth:
        strategy: basic
        user: test@openobserve.com
        password: qweasd