利用 VictoriaMetrics + VictoriaLogs + VictoriaTraces 收集與整理 AWS ElastiCache for Valkey 全棧實戰指南 ¶

# 01-valkey-multi-target-scrape.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMStaticScrape
metadata:
  name: aws-elasticache-valkey-nodes
  namespace: monitoring
spec:
  jobName: "elasticache-valkey"
  targetEndpoints:
    - targets:
        # 填入 ElastiCache for Valkey 各個分片或讀寫節點端點
        - "rediss://prod-valkey-0001-001.xxxxxx.apne1.cache.amazonaws.com:6379"
        - "rediss://prod-valkey-0001-002.xxxxxx.apne1.cache.amazonaws.com:6379"
        - "rediss://prod-valkey-0002-001.xxxxxx.apne1.cache.amazonaws.com:6379"
        - "rediss://prod-valkey-0002-002.xxxxxx.apne1.cache.amazonaws.com:6379"
      interval: "15s"
      scrapeTimeout: "10s"
      path: /scrape
      labels:
        valkey_cluster: "prod-valkey-tokyo"
      relabelConfigs:
        - sourceLabels: [__address__]
          targetLabel: __param_target
        - sourceLabels: [__param_target]
          targetLabel: instance
        # 將實際抓取轉發至 EKS 內部的 redis-exporter 服務
        - targetLabel: __address__
          replacement: "redis-exporter.monitoring.svc.cluster.local:9121"

# 02-valkey-alert-rules.yaml
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: aws-elasticache-valkey-alerts
  namespace: monitoring
spec:
  groups:
    - name: elasticache-valkey.rules
      interval: 30s
      rules:
        # 1. Valkey 引擎主執行緒 CPU 飽和告警 (最關鍵：防止單執行緒阻塞導致全站雪崩)
        - alert: ValkeyEngineCPUHigh
          expr: aws_elasticache_engine_cpuutilization_average > 80
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "ElastiCache Valkey 節點 ({{ $labels.dimension_CacheClusterId }}) 引擎主執行緒 CPU 超過 80%！"
            description: "目前 EngineCPUUtilization 為 {{ $value | printf \"%.1f\" }}%，請立即至 VictoriaLogs 檢查 Slow Log 慢查詢指令。"

        # 2. 記憶體不足引發 Key 強制驅逐告警 (Evictions > 0)
        - alert: ValkeyKeyEvictionDetected
          expr: rate(redis_evicted_keys_total[5m]) > 0
          for: 1m
          labels:
            severity: warning
          annotations:
            summary: "Valkey 叢集 ({{ $labels.valkey_cluster }}) 正在強制驅逐 Key (Evictions)"
            description: "節點 {{ $labels.instance }} 每秒驅逐 {{ $value | printf \"%.2f\" }} 個 Key，代表可用記憶體已觸頂。"

        # 3. 快取命中率過低告警 (低於 85% 可能有快取穿透風險)
        - alert: ValkeyCacheHitRatioLow
          expr: |
            (
              rate(redis_keyspace_hits_total[5m])
              /
              (rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m]) + 1)
            ) * 100 < 85
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Valkey 節點 ({{ $labels.instance }}) 快取命中率低於 85%"

        # 4. 記憶體碎片率過高告警 (超過 1.5 且使用量大於 1GB)
        - alert: ValkeyMemoryFragmentationHigh
          expr: redis_mem_fragmentation_ratio > 1.5 and redis_memory_used_bytes > 1073741824
          for: 15m
          labels:
            severity: info
          annotations:
            summary: "Valkey 節點 ({{ $labels.instance }}) 記憶體碎片率高達 {{ $value }}"

# 03-vector-valkey-logs-to-victorialogs.yaml
transforms:
  enrich_valkey_slowlog:
    type: remap
    inputs:
      - aws_elasticache_logs
    source: |
      .service = "aws-elasticache-valkey"
      .cluster_id = .CacheClusterId
      .duration_us = to_int(."Duration (us)") ?? 0
      .duration_ms = .duration_us / 1000.0
      .client_ip = split(string!(.ClientAddress), ":")[0] ?? ""
      .client_name = .ClientName ?? "unknown"

      # 自動從 Command 拆解出「指令動詞 (如 HGETALL / EVAL)」與「目標 Key」
      cmd_parts = split(string!(.Command), " ")
      .valkey_cmd = upcase(cmd_parts[0] ?? "UNKNOWN")
      .valkey_key = cmd_parts[1] ?? ""
      ._msg = "SlowLog [" + .valkey_cmd + "] took " + to_string(.duration_ms) + "ms: " + string!(.Command)

sinks:
  victorialogs_cluster:
    type: elasticsearch
    inputs:
      - enrich_valkey_slowlog
    endpoints:
      - "http://vlinsert-vl-tokyo-aza.logging.svc.cluster.local:9481/insert/elasticsearch/"
      - "http://vlinsert-vl-tokyo-azc.logging.svc.cluster.local:9481/insert/elasticsearch/"
    api_version: "v8"
    compression: gzip
    query:
      _msg_field: "_msg"
      _time_field: "Timestamp"
      _stream_fields: "service,cluster_id,valkey_cmd"