OpenTelemetry + VictoriaMetrics 全家桶架構與落地實戰指南 ¶

# docker-compose.observability.yml
version: "3.8"

networks:
  obs-net:
    driver: bridge

volumes:
  vmdata:
  vlogsdata:
  vtracesdata:

services:
  # 1. VictoriaMetrics (時序指標資料庫)
  victoria-metrics:
    image: victoriametrics/victoria-metrics:v1.106.0
    container_name: victoria-metrics
    restart: always
    ports:
      - "8428:8428"
    volumes:
      - vmdata:/victoria-metrics-data
    command:
      - "-storageDataPath=/victoria-metrics-data"
      - "-httpListenAddr=:8428"
      - "-retentionPeriod=30d"
      - "-memory.allowedPercent=60"
    networks:
      - obs-net

  # 2. VictoriaLogs (結構化日誌資料庫)
  victoria-logs:
    image: victoriametrics/victoria-logs:v1.53.0
    container_name: victoria-logs
    restart: always
    ports:
      - "9428:9428"
    volumes:
      - vlogsdata:/victoria-logs-data
    command:
      - "-storageDataPath=/victoria-logs-data"
      - "-httpListenAddr=:9428"
      - "-retentionPeriod=14d"
    networks:
      - obs-net

  # 3. VictoriaTraces (分散式鏈路追蹤資料庫)
  victoria-traces:
    image: victoriametrics/victoria-traces:v0.1.0
    container_name: victoria-traces
    restart: always
    ports:
      - "10428:10428" # Web UI / Jaeger Query 相容埠
      - "4317"        # 內部 OTLP gRPC 埠
    volumes:
      - vtracesdata:/victoria-traces-data
    command:
      - "-storageDataPath=/victoria-traces-data"
      - "-httpListenAddr=:10428"
      - "-retentionPeriod=7d"
    networks:
      - obs-net

# /etc/otelcol/config.yaml
receivers:
  # 1. 接收應用程式推送之 OTLP 數據 (Metrics / Logs / Traces)
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

  # 2. 抓取主機節點指標
  hostmetrics:
    collection_interval: 30s
    scrapers:
      cpu:
      memory:
      disk:
      network:
      load:

  # 3. 收集 Kubernetes 容器日誌 (若以 DaemonSet 運行)
  filelog:
    include:
      - /var/log/pods/*/*/*.log
    exclude:
      - /var/log/pods/kube-system_*/*/*.log
    start_at: end
    operators:
      - type: regex_parser
        regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
        timestamp:
          parse_from: attributes.time
          layout: '%Y-%m-%dT%H:%M:%S.%L%z'

processors:
  # 記憶體安全保護，防止大量並發時觸發 OOM
  memory_limiter:
    check_interval: 1s
    limit_percentage: 75
    spike_limit_percentage: 20

  # 批次打包提升網路吞吐與寫入壓縮效率
  batch:
    timeout: 5s
    send_batch_size: 2048

  # 注入 Kubernetes Pod、命名空間與節點等元數據
  k8sattributes:
    auth_type: "serviceAccount"
    passthrough: false
    extract:
      metadata:
        - k8s.pod.name
        - k8s.pod.uid
        - k8s.namespace.name
        - k8s.node.name
        - k8s.deployment.name

  # 使用 OTTL (OpenTelemetry Transformation Language) 進行語意清理
  transform:
    error_mode: ignore
    log_statements:
      - context: log
        statements:
          - set(attributes["env"], "production") where attributes["env"] == nil

exporters:
  # 1. 指標輸出至 VictoriaMetrics (Prometheus Remote Write 協議)
  prometheusremotewrite:
    endpoint: "http://victoria-metrics:8428/api/v1/write"
    resource_to_telemetry_conversion:
      enabled: true # 將 OTel Resource 標籤自動拍平為時序標籤

  # 2. 日誌輸出至 VictoriaLogs (原生 OTLP HTTP 端點)
  otlphttp/victorialogs:
    endpoint: "http://victoria-logs:9428/insert/opentelemetry"
    tls:
      insecure: true

  # 3. 鏈路追蹤輸出至 VictoriaTraces (原生 OTLP gRPC 端點)
  otlp/victoriatraces:
    endpoint: "victoria-traces:10428"
    tls:
      insecure: true

service:
  telemetry:
    logs:
      level: info
  pipelines:
    # 指標處理通道
    metrics:
      receivers: [otlp, hostmetrics]
      processors: [memory_limiter, k8sattributes, batch]
      exporters: [prometheusremotewrite]

    # 日誌處理通道
    logs:
      receivers: [otlp, filelog]
      processors: [memory_limiter, k8sattributes, transform, batch]
      exporters: [otlphttp/victorialogs]

    # 鏈路處理通道
    traces:
      receivers: [otlp]
      processors: [memory_limiter, k8sattributes, batch]
      exporters: [otlp/victoriatraces]

# /etc/vmalert/rules/alert-rules.yml
groups:
  # 1. 基於 VictoriaMetrics 指標之規則
  - name: MetricAlerts
    rules:
      - alert: ApiHigh5xxErrorRate
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[2m])) by (service)
            /
            sum(rate(http_requests_total[2m])) by (service)
          ) * 100 > 5
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "服務 {{ $labels.service }} 錯誤率高於 5%"
          description: "近 2 分鐘內 HTTP 5xx 比例突破 SLA 臨界線。"

  # 2. 基於 VictoriaLogs 日誌之規則 (LogsQL 報警)
  - name: LogAlerts
    rules:
      - alert: DatabaseConnectionPoolExhausted
        expr: |
          count_over_time(_time:2m "connection pool exhausted" OR "deadlock detected")[2m] > 10
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: "資料庫連線池枯竭警告"
          description: "在過去 2 分鐘內日誌中累計超過 10 次連線池耗盡事件。"