批调度：Kueue、Volcano 与 Gang 调度

集群剩余 16 张卡，同时来了两个任务，各要 4 个 Pod × 4 卡 = 16 卡

逐个调度（默认调度器）            Gang 调度
  任务 A: ■ ■ □ □  (2/4 Running)     任务 A: ■ ■ ■ ■  全部 Running
  任务 B: ■ ■ □ □  (2/4 Running)     任务 B: □ □ □ □  整组排队
  → 两边都凑不齐，卡全被占着空等     → A 跑完释放后 B 再整组启动

用户提交 Job（带 queue-name 标签，suspend: true）
        │
        ▼
Kueue webhook 为 Job 创建一个 Workload 对象，记录它总共需要多少资源
        │
        ▼
Workload 进入 LocalQueue → ClusterQueue 排队
        │
   配额够吗？ ── 否 ──→ 继续排队（0 个 Pod）
        │是
        ▼
Kueue 把 Job 的 suspend 改为 false，并按 ResourceFlavor 注入 nodeSelector/tolerations
        │
        ▼
Job 控制器创建 Pod → kube-scheduler 正常调度

apiVersion: kueue.x-k8s.io/v1beta2
kind: ResourceFlavor
metadata:
  name: h100
spec:
  nodeLabels:
    example.com/gpu: h100                 # 只选 H100 节点
  tolerations:                             # GPU 节点带污点，这里显式声明容忍
    - key: "nvidia.com/gpu"
      operator: "Exists"
      effect: "NoSchedule"
---
apiVersion: kueue.x-k8s.io/v1beta2
kind: ClusterQueue
metadata:
  name: gpu-cluster-queue
spec:
  namespaceSelector: {}                    # 所有命名空间都可以使用
  resourceGroups:
    - coveredResources: ["cpu", "memory", "nvidia.com/gpu"]
      flavors:
        - name: h100
          resources:                       # 顺序必须与 coveredResources 一致
            - name: "cpu"
              nominalQuota: "382"
            - name: "memory"
              nominalQuota: 3000Gi
            - name: "nvidia.com/gpu"
              nominalQuota: 24             # 3 节点 × 8 卡
  preemption:
    withinClusterQueue: Never
    reclaimWithinCohort: Never
---
apiVersion: kueue.x-k8s.io/v1beta2
kind: LocalQueue
metadata:
  name: gpu-local-queue
  namespace: default
spec:
  clusterQueue: gpu-cluster-queue
---
apiVersion: kueue.x-k8s.io/v1beta2
kind: WorkloadPriorityClass
metadata:
  name: high-priority
value: 10000

apiVersion: batch/v1
kind: Job
metadata:
  generateName: sample-job-
  namespace: default
  labels:
    kueue.x-k8s.io/queue-name: gpu-local-queue
    kueue.x-k8s.io/priority-class: high-priority     # 可选
spec:
  suspend: true              # 交给 Kueue 决定何时开始
  parallelism: 3
  completions: 3
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: worker
          image: registry.k8s.io/e2e-test-images/agnhost:2.53
          args: ["pause"]
          resources:
            requests:
              cpu: "1"
              memory: 200Mi

kubectl get workloads -n default
kubectl describe workload <name> -n default     # Events 和 conditions 会写明为什么没被准入
kubectl get clusterqueue gpu-cluster-queue      # PENDING WORKLOADS / ADMITTED WORKLOADS

apiVersion: kueue.x-k8s.io/v1beta2
kind: ClusterQueue
metadata:
  name: team-a
spec:
  cohortName: ai-platform
  namespaceSelector:
    matchLabels:
      team: a
  resourceGroups:
    - coveredResources: ["nvidia.com/gpu"]
      flavors:
        - name: h100
          resources:
            - name: "nvidia.com/gpu"
              nominalQuota: 16          # 保底 16 卡
              borrowingLimit: 8         # 最多向别人借 8 卡
  preemption:
    reclaimWithinCohort: Any            # 别人借了我的卡，我需要时可以抢回来
    withinClusterQueue: LowerPriority   # 队列内高优先级可以抢占低优先级

apiVersion: batch.volcano.sh/v1alpha1
kind: Job
metadata:
  name: mpi-demo
spec:
  minAvailable: 3               # Gang：至少 3 个 Pod 能同时放下才调度
  schedulerName: volcano
  queue: default
  tasks:
    - name: worker
      replicas: 3
      template:
        spec:
          restartPolicy: Never
          containers:
            - name: worker
              image: registry.k8s.io/pause:3.10
              resources:
                requests:
                  cpu: "1"

apiVersion: scheduling.volcano.sh/v1beta1
kind: PodGroup
metadata:
  name: my-training
spec:
  minMember: 2          # 必须等于训练节点数
  queue: default

apiVersion: scheduling.x-k8s.io/v1alpha1
kind: PodGroup
metadata:
  name: pg1
spec:
  scheduleTimeoutSeconds: 10
  minMember: 3                  # 至少 3 个 Pod 才能一起调度
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: pause
spec:
  replicas: 2                   # 故意只给 2 个
  selector:
    matchLabels:
      app: pause
  template:
    metadata:
      labels:
        app: pause
        scheduling.x-k8s.io/pod-group: pg1
    spec:
      schedulerName: scheduler-plugins-scheduler
      containers:
        - name: pause
          image: registry.k8s.io/pause:3.10