多租户与平台工程

隔离强度    弱 ─────────────────────────────────────────────→ 强

           命名空间租户           虚拟集群                 托管控制平面            独立集群
           Namespace + RBAC      vcluster / k3k          Kamaji                 每租户一套
           + Quota + NetPol      租户有自己的 apiserver，  租户有独立控制平面        完整集群
                                 Pod 跑在宿主集群节点上    和独立的工作节点
共享       控制平面、节点、CRD    节点、CNI、存储           管理集群（跑控制平面）     几乎不共享

apiVersion: v1
kind: ResourceQuota
metadata:
  name: team-a-quota
  namespace: team-a
spec:
  hard:
    requests.cpu: "64"
    requests.memory: 512Gi
    limits.memory: 512Gi
    requests.nvidia.com/gpu: "8"
    persistentvolumeclaims: "20"

租户 kubectl ──→ vcluster apiserver（在宿主 ns: vc-team-a 里）
                        │
                   syncer 把"低层资源"同步到宿主集群
                        ▼
宿主集群 ns: vc-team-a
   Pod: web-x-default-x-vcluster        ← 真正跑在宿主节点上
   Service、PVC、ConfigMap（被引用的）   ← 同步下来
   Deployment、CRD、RBAC                ← 不同步，只存在于虚拟集群里

controlPlane:
  distro:
    k8s:
      enabled: true                 # 使用标准 K8s 组件，而不是 k3s
  backingStore:
    etcd:
      deploy:
        enabled: true               # 独立 etcd，3 副本
        statefulSet:
          highAvailability:
            replicas: 3
  statefulSet:
    highAvailability:
      replicas: 2
  coredns:
    deployment:
      replicas: 2
  service:
    spec:
      type: LoadBalancer            # 由 MetalLB 分配 IP，租户直接访问
  proxy:
    extraSANs:
      - team-a.k8s.example.com

exportKubeConfig:
  context: team-a
  server: https://team-a.k8s.example.com:443

sync:
  toHost:
    pods:
      enabled: true
  fromHost:
    nodes:
      enabled: true                 # 让租户看到真实节点（及 GPU 容量）
      selector:
        labels:
          node-pool: team-a         # 只同步分给该租户的节点
    storageClasses:
      enabled: true

policies:
  resourceQuota:
    enabled: true
    quota:
      requests.cpu: 128
      requests.memory: 1Ti
      requests.nvidia.com/gpu: 16
  limitRange:
    enabled: true
    default:
      cpu: "2"
      memory: 2Gi
    defaultRequest:
      cpu: 100m
      memory: 128Mi
  podSecurityStandard: baseline
  networkPolicy:
    enabled: true

helm repo add loft https://charts.loft.sh
helm upgrade --install team-a loft/vcluster \
  -n vc-team-a --create-namespace --version 0.33.1 -f vcluster-values.yaml

# 取出 kubeconfig 交给租户
kubectl get secret vc-team-a -n vc-team-a --template='{{.data.config}}' | base64 -d > team-a.kubeconfig
kubectl --kubeconfig team-a.kubeconfig get nodes

apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
  name: team-b
  namespace: k3k-team-b
spec:
  mode: shared
  version: v1.34.1-k3s1
  servers: 3
  agents: 1
  persistence:
    type: dynamic
  tlsSANs:
    - team-b.k8s.example.com
  expose:
    ingress:
      ingressClassName: nginx
      annotations:
        nginx.ingress.kubernetes.io/ssl-passthrough: "true"

apiVersion: kamaji.clastix.io/v1alpha1
kind: TenantControlPlane
metadata:
  name: tenant-c
  namespace: tenants
spec:
  controlPlane:
    deployment:
      replicas: 3
    service:
      serviceType: LoadBalancer
  kubernetes:
    version: v1.33.4
    kubelet:
      cgroupfs: systemd
    admissionControllers:
      - ResourceQuota
      - LimitRanger
  networkProfile:
    port: 6443
    certSANs:
      - tenant-c.k8s.example.com
    serviceCidr: 10.96.0.0/16
    podCidr: 10.244.0.0/16
    dnsServiceIPs:
      - 10.96.0.10
  addons:
    coreDNS: {}
    kubeProxy: {}
    konnectivity:
      server:
        port: 8132          # 控制平面与租户节点之间的反向隧道

kubectl get tcp -n tenants        # tcp 是 TenantControlPlane 的短名
kubectl get secret tenant-c-admin-kubeconfig -n tenants \
  -o jsonpath='{.data.admin\.conf}' | base64 -d > tenant-c.kubeconfig

# 针对租户集群生成 join 命令，再到租户的机器上执行输出的 kubeadm join
kubeadm --kubeconfig=tenant-c.kubeconfig token create --print-join-command

platform-gitops/
├── infrastructure/              # 平台组件：gpu-operator、kueue、kgateway ...
│   ├── base/
│   └── clusters/prod/
├── tenants/
│   ├── _template/               # 租户标配：ns、quota、rbac、netpol、localqueue
│   ├── team-a/                  # kustomization 引用模板 + 覆盖配额
│   └── team-b/
└── vclusters/
    ├── team-a/values.yaml
    └── partner-x/values.yaml