可观测性：指标、日志与事件

                    ┌──────────────┐
 node-exporter ─┐   │              │     ┌──────────┐
 kube-state-  ──┼──▶│   vmagent    │────▶│ vmsingle │◀── Grafana（看图）
   metrics      │   │ (抓取/转发)   │     │ (存储)    │
 kubelet/cAdvisor┤   └──────────────┘     └────┬─────┘
 apiserver/etcd ─┘          ▲                  │
                            │             ┌────▼────┐     ┌──────────────┐
       VMServiceScrape / ServiceMonitor   │ vmalert │────▶│ Alertmanager │──▶ 飞书/邮件/Webhook
       （声明抓取目标）                    │(规则计算)│     └──────────────┘
                                          └─────────┘

releases:
  - name: vm
    namespace: vm
    create_namespace: true
    chart:
      name: oci://ghcr.io/victoriametrics/helm-charts/victoria-metrics-k8s-stack
      version: 0.70.0   # 截至本文写作时的版本，以官方发布为准
    values:
      - values.yml

vmsingle:
  spec:
    retentionPeriod: 30d            # 保留 30 天
    storage:
      storageClassName: shared-cs   # 换成集群里实际可用的 StorageClass
      resources:
        requests:
          storage: 1Ti
    extraArgs:
      search.maxQueryDuration: 1m
      search.maxConcurrentRequests: '120'
      maxLabelsPerTimeseries: "1024"   # 默认 label 数上限较低，pod 全量 label 会被截断
      memory.allowedPercent: '90'      # 容器内由 limits 兜底，不必给宿主机留余量

victoria-metrics-operator:
  operator:
    disable_prometheus_converter: false  # 开启 Prometheus CRD 转换
    enable_converter_ownership: true     # 删除 ServiceMonitor 时联动删除转换出来的对象

kube-state-metrics:
  metricLabelsAllowlist:
    - nodes=[node.kubernetes.io/cpu,nvidia.com/gpu,node-role.kubernetes.io/maintenance]
    - pods=[*]

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
  - https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.88.1/stripped-down-crds.yaml

helmwave up --build
kubectl apply -k prometheus

replicas: 2
persistence:
  enabled: true
  size: 100Gi
tolerations:
  - operator: "Exists"
    effect: "NoSchedule"
affinity:
  nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
      nodeSelectorTerms:
        - matchExpressions:
            - key: node-role.kubernetes.io/control-plane
              operator: Exists

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: demo-app
  namespace: demo
spec:
  selector:
    matchLabels:
      app: demo-app          # 选中 Service 的 label，不是 Pod 的
  endpoints:
    - port: metrics          # Service 里端口的 name，不是数字
      interval: 30s
      path: /metrics

apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
metadata:
  name: dcgm-exporter
  namespace: gpu-operator
spec:
  namespaceSelector:
    matchNames: [gpu-operator]
  selector:
    matchLabels:
      app: nvidia-dcgm-exporter
  endpoints:
    - port: gpu-metrics

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cluster-extra
  namespace: vm
spec:
  groups:
    - name: cluster-extra.rules
      rules:
        - alert: EtcdDbNearQuota
          expr: etcd_mvcc_db_total_size_in_bytes / etcd_server_quota_backend_bytes > 0.8
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "etcd {{ $labels.instance }} DB 已用配额 {{ $value | humanizePercentage }}"
            description: "超过配额会触发 NOSPACE alarm，集群拒绝写入。尽快 compact + defrag。"
        - alert: NodeDiskAlmostFull
          expr: |
            node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"}
              / node_filesystem_size_bytes < 0.10
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.instance }} {{ $labels.mountpoint }} 可用空间低于 10%"
        - alert: PodCrashLooping
          expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}[5m]) >= 1
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} 持续 CrashLoopBackOff"

Pod stdout/stderr ─▶ /var/log/pods/<ns>_<pod>_<uid>/<container>/*.log
                                    │
                          采集器 DaemonSet（promtail / fluent-bit）
                                    │  附加 namespace、pod、node 等标签
                                    ▼
                         Loki / VictoriaLogs  ◀── Grafana 查询

deploymentMode: SimpleScalable
loki:
  schemaConfig:
    configs:
      - from: 2024-04-01
        store: tsdb
        object_store: s3
        schema: v13
        index: { prefix: loki_index_, period: 24h }
  storage:
    type: s3
    s3:
      endpoint: "s3.example.com"
      accessKeyId: "..."
      secretAccessKey: "..."
      s3ForcePathStyle: true
  limits_config:
    ingestion_rate_mb: 20
    ingestion_burst_size_mb: 100
    per_stream_rate_limit: 6MB        # 默认 3MB，日志量大的 Pod 会被限流丢日志
    per_stream_rate_limit_burst: 50MB
    max_line_size: 256kb
    max_line_size_truncate: true      # 超长行截断而不是整行丢弃

server:
  persistentVolume:
    enabled: true
    size: 2Ti
  extraArgs:
    retentionPeriod: 60d
  resources:
    requests: { cpu: 500m, memory: 1Gi }
    limits:   { cpu: 8, memory: 8Gi }
fluent-bit:
  enabled: true
  tolerations:
    - effect: NoSchedule
      operator: Exists        # GPU 节点、控制平面节点通常带污点，漏掉就收不到这些节点的日志

env:
  GF_INSTALL_PLUGINS: https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.3.0/victorialogs-datasource-v0.3.0.zip;victorialogs-datasource
  GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: victorialogs-datasource

logLevel: info
logFormat: json
maxEventAgeSeconds: 60
route:
  routes:
    - match:
        - receiver: "victorialogs"
receivers:
  - name: "victorialogs"
    webhook:
      endpoint: http://vl-victoria-logs-single-server:9428/insert/jsonline?_stream_fields=cluster,classify,namespace&_msg_field=msg&_time_field=time
      layout:
        cluster:   "{{ .ClusterName }}"
        classify:  "k8s-event"
        time:      "{{ .FirstTimestamp.Format \"2006-01-02T15:04:05Z\" }}"
        namespace: "{{ .InvolvedObject.Namespace }}"
        msg:       "{{ .Message }}"
        type:      "{{ .Type }}"
        reason:    "{{ .Reason }}"
        count:     "{{ .Count }}"
        kind:      "{{ .InvolvedObject.Kind }}"
        name:      "{{ .InvolvedObject.Name }}"
        host:      "{{ .Source.Host }}"