Layer 1: scenarios-as-modules (Cl2Benchmark archetype + Kata demo + cycle gate)

.claude/skills/generate-yaml/SKILL.md

@@ -5,6 +5,24 @@ description: Generate pipeline YAML from KCL source
 
 # Generate pipeline YAML from KCL source
 
+## Validate scenarios first
+
+Before generating, run the cross-scenario validation gate. It catches the one
+thing KCL's compiler cannot see: **path-string references** between scenarios
+(one scenario's `cl2Manifest` / `kwokNodeManifest` pointing into another
+scenario's directory) that form a loop. KCL already rejects circular *imports*
+on its own, so this gate deliberately does not re-check those.
+
+```bash
+python3 scripts/validate_scenarios.py
+```
+
+On success it prints `validate_scenarios: OK ...` and exits 0. If it fails
+(exit 1), it prints the offending cycle (`a -> b -> a`); fix the path-string
+references before continuing.
+
+## Generate
+
 Given a pipeline KCL file (e.g. `path/to/pipeline.k`), determine its directory and run:
 
 ```bash
@@ -18,6 +36,17 @@ The output YAML file is written alongside the KCL source file in the same direct
 kcl run kcl/example_pipeline/pipeline.k -S output -o kcl/example_pipeline/pipeline.yaml
 ```
 
+### If `kcl run` reports a circular import
+
+KCL fails fast with `error[E1001] RecursiveLoad` / `circular reference between
+modules ...` when scenarios import each other in a loop. To reframe that raw
+diagnostic in scenario terms for the author, pass the captured stderr through:
+
+```python
+from scripts.validate_scenarios import format_kcl_cycle_error
+print(format_kcl_cycle_error(stderr))  # None if stderr is not a RecursiveLoad
+```
+
 ## Split if oversized
 
 Azure DevOps enforces a 2 MB limit on a single pipeline YAML file. After generating the YAML, check its size:

kcl/example_pipeline/pipeline.k

@@ -1,22 +1,11 @@
-import azure_pipelines.ap
-import azure_pipelines.ap.jobs.job
-import lib.const
+import lib.scenario
 import lib.steps.azure
-import lib.steps.common
-import lib.steps.k8s
-import lib.util
 
-SUBSCRIPTION_ID = const.DEFAULT_SUBSCRIPTION_ID
-RESOURCE_GROUP = "$(RUN_ID)"
 LOCATION = "westus2"
 CLUSTER = "stg-H2-yaolei"
-CL2_POOL = "cl2pool"
-CL2_TAINT_PREFIX = "cl2pool"
-CL2_NAMESPACE = "clusterloader2"
-KWOK_NODE_COUNT = 100
 
 # SkipAKSCluster disables Gatekeeper on ephemeral clusters
-requestBody = util.escapeStr("""
+requestBody = """
 {
     "location": "${LOCATION}",
     "identity": { "type": "SystemAssigned" },
@@ -57,111 +46,31 @@ requestBody = util.escapeStr("""
         }
       }
     }
-  }""")
-
-createClusterScript = """
-az cloud update --endpoint-resource-manager https://eastus2euap.management.azure.com/
-az rest \\
-  --method put \\
-  --uri "/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP}/providers/Microsoft.ContainerService/managedClusters/${CLUSTER}?api-version=2026-01-02-preview" \\
-  --body "${requestBody}"
-"""
+  }"""
 
 cl2ResultJson = """{
     "cl2_status": "\${STATUS}", 
     "region": "${LOCATION}"
   }"""
 
-output = ap.Pipeline {
+output = scenario.build(scenario.Cl2Benchmark {
     name = "Example Pipeline"
+    location = LOCATION
+    cluster = CLUSTER
+    requestBody = requestBody
 
-    trigger =  ["v2"]
-    pool = const.DEFAULT_POOL
-
-    jobs = [
-        job.Job {
-            job = "benchmarking"
-            displayName = "Benchmarking"
-            timeoutInMinutes = 1440
+    nodePool = azure.NodePool {
+        name = "cl2pool"
+        sku = "Standard_D8S_v4"
+        count = 4
+        taintPrefix = "cl2pool"
+    }
 
-            steps = [
-                common.SetRunId(),
-                common.InstallPythonDependencies(),
-                azure.Login(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    SUBSCRIPTION_ID,
-                    LOCATION
-                ),
-                azure.CreateResourceGroup(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    RESOURCE_GROUP,
-                    LOCATION,
-                    SUBSCRIPTION_ID
-                ),
-                azure.AzCli(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    "Create cluster",
-                    createClusterScript),
-                azure.WaitForClusterSucceeded(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    CLUSTER,
-                    RESOURCE_GROUP,
-                    SUBSCRIPTION_ID),
-                azure.CreateNodePool(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    CLUSTER,
-                    RESOURCE_GROUP,
-                    SUBSCRIPTION_ID,
-                    azure.NodePool {
-                      name = CL2_POOL,
-                      sku = "Standard_D8S_v4",
-                      count = 4,
-                      taintPrefix = CL2_TAINT_PREFIX }),
-                azure.GetCredentials(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    CLUSTER,
-                    RESOURCE_GROUP,
-                    SUBSCRIPTION_ID),
-                *k8s.CreateKwokNodes(
-                  KWOK_NODE_COUNT,
-                  params = {
-                    "node-manifest-path": "$(Pipeline.Workspace)/s/kcl/example_pipeline/kwok-node.yaml"
-                  }),
-                k8s.RunClusterLoader2(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    CL2_NAMESPACE,
-                    manifest = "kcl/example_pipeline/cl2.yaml"),
-                k8s.PrintCl2PodLogs(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    CL2_NAMESPACE),
-                azure.AzCli(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    "Collect cl2 pods status",
-                    """
-TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-STATUS="succeeded"
+    kwokNodeCount = 100
+    kwokNodeManifest = "$(Pipeline.Workspace)/s/kcl/example_pipeline/kwok-node.yaml"
 
-# Check if any cl2 pod failed
-FAILED=$(kubectl get pods --namespace="${CL2_NAMESPACE}" -l job-name=cl2 -o jsonpath='{range .items[*]}{.status.phase}{"\\n"}{end}' | grep -c "^Failed$" || true)
-if [ "$FAILED" -gt 0 ]; then
-  STATUS="failed"
-fi
+    cl2Namespace = "clusterloader2"
+    cl2Manifest = "kcl/example_pipeline/cl2.yaml"
 
-cat > /tmp/run-result.json << EOF
-${util.formatResult(cl2ResultJson)}
-EOF
-cat /tmp/run-result.json
-"""),
-                common.UploadResult(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    const.DEFAULT_STORAGE_ACCOUNT,
-                    const.DEFAULT_STORAGE_SUBSCRIPTION_ID,
-                    const.DEFAULT_STORAGE_CONTAINER),
-                azure.DeleteResourceGroup(
-                    const.DEFAULT_SERVICE_CONNECTION,
-                    RESOURCE_GROUP,
-                    SUBSCRIPTION_ID)
-            ]
-        }
-    ]
-}
+    resultJson = cl2ResultJson
+})

kcl/example_pipeline/pipeline.yaml

@@ -39,10 +39,14 @@ jobs:
       azureSubscription: Azure-for-Telescope-internal
       scriptType: bash
       scriptLocation: inlineScript
-      inlineScript: |
+      inlineScript: |+
         set -exo pipefail
 
-        az group create --name "$(RUN_ID)" --location "westus2" --subscription "b8ceb4e5-f05b-4562-a9f5-14acb1f24219"
+        az group create \
+          --name "$(RUN_ID)" \
+          --location "westus2" \
+          --subscription "b8ceb4e5-f05b-4562-a9f5-14acb1f24219" \
+
     displayName: Create resource group in westus2 (b8ceb4e5-f05b-4562-a9f5-14acb1f24219)
   - task: AzureCLI@2
     inputs:

kcl/kata_benchmark/cl2.k

@@ -0,0 +1,17 @@
+import lib.scenario
+
+# cl2.k renders the ClusterLoader2 driver manifest for the Kata benchmark.
+#
+# Enabling AKS Kata Confidential Containers for the measured workload is a
+# single field: `runtimeClassName`. The archetype's manifest builder templates
+# it into the override ConfigMap as CL2_RUNTIME_CLASS_NAME, which the in-cluster
+# CL2 image stamps onto every workload pod it generates. Everything else inherits
+# the shared defaults, so this file stays a couple of lines.
+#
+# Generate with:
+#   kcl run kcl/kata_benchmark/cl2.k -S manifests -o kcl/kata_benchmark/cl2.yaml
+manifests = scenario.buildCl2Manifest(scenario.Cl2Manifest {
+    override = scenario.Cl2Override {
+        runtimeClassName = "kata-mshv-vm-isolation"
+    }
+})

kcl/kata_benchmark/cl2.yaml

@@ -0,0 +1,92 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cl2-override
+  namespace: clusterloader2
+data:
+  override.yaml: |-
+    NODES: 30
+    PODS_PER_NODE: 10
+    BIG_GROUP_SIZE: 8
+    MEDIUM_GROUP_SIZE: 8
+    SMALL_GROUP_SIZE: 4
+    SMALL_STATEFUL_SETS_PER_NAMESPACE: 0
+    MEDIUM_STATEFUL_SETS_PER_NAMESPACE: 0
+    CL2_RATE_LIMIT_POD_CREATION: false
+    CL2_ENABLE_PVS: false
+    CL2_ENABLE_CLUSTER_OOMS_TRACKER: false
+    CL2_DEFAULT_QPS: 5000
+    CL2_RUN_ON_ARM_NODES: true # This is a hack to allow Cl2 to run on Kwok nodes
+    CL2_RUNTIME_CLASS_NAME: kata-mshv-vm-isolation
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cl2
+  namespace: clusterloader2
+spec:
+  completions: 4
+  parallelism: 4
+  backoffLimit: 0
+  template:
+    spec:
+      containers:
+      - name: cl2
+        image: ghcr.io/azure/clusterloader2:v20260220
+        args:
+        - '--provider=aks'
+        - '--run-from-cluster=true'
+        - '--v=2'
+        - '--testoverrides=/override/override.yaml'
+        - '--testconfig=testing/load/config.yaml'
+        - '--k8s-clients-number=500'
+        resources:
+          requests:
+            cpu: '6'
+            memory: '24Gi'
+        volumeMounts:
+        - mountPath: /override
+          name: cl2-override
+      nodeSelector:
+        agentpool: cl2pool
+      restartPolicy: Never
+      serviceAccountName: cl2
+      tolerations:
+      - effect: NoSchedule
+        key: cl2pool
+        operator: Exists
+      volumes:
+      - name: cl2-override
+        configMap:
+          name: cl2-override
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cl2
+  namespace: clusterloader2
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cl2
+rules:
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cl2
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cl2
+subjects:
+- kind: ServiceAccount
+  name: cl2
+  namespace: clusterloader2

kcl/kata_benchmark/kwok-node.yaml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: Node
+metadata:
+  name: {{node_name}}
+  annotations:
+    node.alpha.kubernetes.io/ttl: "0"
+    kwok.x-k8s.io/node: fake
+  labels:
+    beta.kubernetes.io/arch: amd64
+    beta.kubernetes.io/os: linux
+    kubernetes.io/arch: amd64
+    kubernetes.io/hostname: {{node_name}}
+    kubernetes.io/os: linux
+    kubernetes.io/role: agent
+    node-role.kubernetes.io/agent: ""
+    type: kwok
+spec:
+  providerID: "kwok://{{node_name}}"
+  unschedulable: false
+  taints: # Avoid scheduling actual running pods to fake Node
+    - effect: NoSchedule
+      key: kubernetes.io/arch
+      value: arm64 # This is a hack to allow Cl2 pods to run on Kwok nodes.
+status:
+  addresses:
+    - type: InternalIP
+      address: {{node_ip}}
+  allocatable:
+    cpu: {{node_cpu}}
+    memory: {{node_memory}}
+    pods: {{node_pods}}
+    nvidia.com/gpu: {{node_gpu}}
+  capacity:
+    cpu: {{node_cpu}}
+    memory: {{node_memory}}
+    pods: {{node_pods}}
+    nvidia.com/gpu: {{node_gpu}}
+  conditions:
+    - type: "Ready"
+      status: "True"
+      reason: "KubeletReady"
+      message: "kubelet is posting ready status"
+  nodeInfo:
+    architecture: amd64
+    bootID: ""
+    containerRuntimeVersion: ""
+    kernelVersion: ""
+    kubeProxyVersion: fake
+    kubeletVersion: fake
+    machineID: ""
+    operatingSystem: linux
+    osImage: ""
+    systemUUID: ""
+  phase: Running