生产网络：Cilium、MetalLB 与证书

network/
├── cilium/
│   ├── helmwave.yml        # 装哪个 Chart、哪个版本、装到哪个命名空间
│   └── values.yml          # Chart 参数，只写和默认值不同的部分
├── metallb/
│   ├── helmwave.yml
│   ├── values.yml
│   ├── kustomization.yaml  # Chart 之外的自定义资源（地址池等）
│   └── ipaddresspool.yaml
└── cert-manager/
    ├── helmwave.yml
    ├── values.yml
    ├── kustomization.yaml
    └── clusterissuer.yaml

project: metallb

repositories:
  - name: metallb
    url: https://metallb.github.io/metallb

releases:
  - name: metallb
    namespace: metallb-system
    create_namespace: true
    chart:
      name: metallb/metallb
      version: 0.15.3
    values:
      - values.yml
    wait: true

cd network/metallb
helmwave up --build            # 安装/升级 Chart
kubectl apply -k .             # 应用 Chart 之外的 CR（依赖 Chart 带的 CRD，所以放第二步）

# apiserver 地址：指向 Kubespray 在每个节点上的本地 nginx 代理
k8sServiceHost: "127.0.0.1"
k8sServicePort: "6443"

# 保留 kube-proxy，Cilium 只负责 Pod 网络和策略
kubeProxyReplacement: "false"

ipam:
  mode: cluster-pool
  operator:
    clusterPoolIPv4PodCIDRList: ["172.24.0.0/13"]   # 与 kube_pods_subnet 一致
    clusterPoolIPv4MaskSize: 24                     # 与 kube_network_node_prefix 一致

cni:
  chainingMode: portmap   # 配合 kube-proxy 支持 hostPort
  exclusive: false        # 允许共存其他 CNI（如 Multus、Spiderpool 做 RDMA 网卡）

bpf:
  enableTCX: false        # 规避 TCX 程序残留的问题，见下文

operator:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: node-role.kubernetes.io/control-plane
                operator: Exists

kubectl -n kube-system rollout status ds/cilium
kubectl get nodes
cilium status          # 需要单独安装 cilium CLI

kubectl -n kube-system rollout restart deployment/cilium-operator

controller:
  affinity: &cp-affinity
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: node-role.kubernetes.io/control-plane
                operator: Exists
  tolerations: &cp-tolerations
    - operator: Exists
speaker:
  affinity: *cp-affinity
  tolerations: *cp-tolerations
  ignoreExcludeLB: true   # 控制平面节点默认带"排除外部 LB"的标签，这里忽略它

apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: default
  namespace: metallb-system
spec:
  addresses:
    - 192.168.10.200-192.168.10.210   # 向网管申请、不在 DHCP 范围内的地址
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: default
  namespace: metallb-system
spec:
  ipAddressPools: ["default"]

metadata:
  annotations:
    metallb.io/loadBalancerIPs: 192.168.10.210

# 1. 这个 IP 当前由哪个节点宣告
kubectl -n metallb-system get servicel2statuses.metallb.io

# 2. speaker 日志里有没有宣告记录
kubectl -n metallb-system logs -l app.kubernetes.io/component=speaker | grep serviceAnnounced

# 3. 在同网段的另一台机器上看 ARP 是否有回应，MAC 是否是宣告节点的
arping -I eth0 192.168.10.210

# 4. 在宣告节点上抓 ARP 包
tcpdump -ni eth0 arp and host 192.168.10.210

crds:
  enabled: true          # 让 Chart 安装 CRD
replicaCount: 2
podDisruptionBudget:
  enabled: true
webhook:
  replicaCount: 2
  podDisruptionBudget:
    enabled: true
cainjector:
  replicaCount: 2
  podDisruptionBudget:
    enabled: true

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: ops@example.com
    privateKeySecretRef:
      name: letsencrypt-prod
    solvers:
      - http01:
          gatewayHTTPRoute:    # 需在 values 中设置 config.enableGatewayAPI: true；用 Ingress 时改为 ingress.ingressClassName
            parentRefs:
              - name: main-gateway
                namespace: istio-system
                kind: Gateway

kubectl get certificate,certificaterequest,order,challenge -A
kubectl describe challenge -n <ns> <name>    # 通常能看到具体的失败原因

apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: main-gateway
  namespace: istio-system
  annotations:
    metallb.io/loadBalancerIPs: 192.168.10.210
spec:
  gatewayClassName: istio
  listeners:
    - name: https
      hostname: "*.dev1.bj1.example.com"
      port: 443
      protocol: HTTPS
      tls:
        certificateRefs:
          - name: wildcard-dev1-tls
      allowedRoutes:
        namespaces:
          from: Selector
          selector:
            matchLabels:
              gateway-access: "true"   # 只有打了这个标签的命名空间能挂路由

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: grafana
  namespace: monitoring
spec:
  parentRefs:
    - name: main-gateway
      namespace: istio-system
  hostnames: ["gf.dev1.bj1.example.com"]
  rules:
    - backendRefs:
        - name: grafana
          port: 80

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: allow-github-only
  namespace: ci
spec:
  endpointSelector: {}
  egress:
    - toFQDNs:
        - matchName: github.com
        - matchPattern: "*.githubusercontent.com"
    # 必须放行 DNS，并让 Cilium 代理 DNS 查询，才能学到域名对应的 IP
    - toEndpoints:
        - matchLabels:
            k8s:io.kubernetes.pod.namespace: kube-system
            k8s-app: kube-dns
      toPorts:
        - ports: [{ port: "53", protocol: ANY }]
          rules:
            dns: [{ matchPattern: "*" }]
    # 开了 nodelocaldns 时，DNS 实际发往本机
    - toEntities: ["host"]
      toPorts:
        - ports: [{ port: "53", protocol: ANY }]
          rules:
            dns: [{ matchPattern: "*" }]