From e361b4b69fdac4519f40e79db95404e351e7aa19 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 6 May 2026 15:10:53 -0400 Subject: [PATCH 1/2] k8s: add nemo-training deployment for LoRA fine-tuning and nemo2riva MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interactive training pod on its own H100. Scale-to-zero by default (replicas: 0); user scales up manually via `oc scale`, shells in with `oc rsh`, runs training or the nemo2riva conversion, then scales back to 0 to stop H100 billing. - Image: nvcr.io/nvidia/nemo:25.04 (pulled from nvcr.io via the namespace ngc-secret linked to the default SA). - Command: sleep infinity — keeps the pod alive for interactive use. No Service, no Route — network-internal only. - Mounts: vllm-model-cache at /models (shared; also where .riva output files land for the riva-stt deployment to pick up) and nemo-training-data at /data (training-only). - Resources: 4-16 CPU, 32-64Gi RAM, 1x H100. Heavier RAM than serving pods because DataLoader workers + dataset shards + checkpoint staging. - 8Gi tmpfs at /dev/shm for PyTorch DataLoader IPC. - Standard labels + purpose annotation, matching the PVC template. Closes #8 Part of #5 Co-Authored-By: Claude Opus 4.7 (1M context) --- k8s/nemo-training-deployment.yaml | 97 +++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 k8s/nemo-training-deployment.yaml diff --git a/k8s/nemo-training-deployment.yaml b/k8s/nemo-training-deployment.yaml new file mode 100644 index 0000000..b3c0891 --- /dev/null +++ b/k8s/nemo-training-deployment.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nemo-training + labels: + app: nemo-training + app.kubernetes.io/part-of: accent-stt + app.kubernetes.io/component: training + annotations: + purpose: nemo-lora-finetuning-and-nemo2riva-conversion +spec: + # Scale-to-zero is deliberate: fine-tuning is a batch workload that + # only runs during an active session. User scales up manually, shells + # in via `oc rsh deployment/nemo-training`, runs training or the + # nemo2riva conversion, then scales back to 0. H100 is ~$3/hr — do + # not leave this running. + replicas: 0 + selector: + matchLabels: + app: nemo-training + template: + metadata: + labels: + app: nemo-training + spec: + # H100 taint, matching every existing deployment in this repo. + tolerations: + - key: nvidia.com/gpu.product + operator: Equal + value: NVIDIA-H100-80GB-HBM3 + effect: NoSchedule + containers: + - name: nemo + # NVIDIA NeMo Framework container. Verify the latest tag before + # applying: + # skopeo list-tags docker://nvcr.io/nvidia/nemo + # Bump to the newest stable monthly release if available. + # Requires the namespace `ngc-secret` pull secret linked to the + # default SA — see CLAUDE.md § NGC Registry Access. + image: nvcr.io/nvidia/nemo:25.04 + command: ["/bin/sh", "-c"] + # sleep infinity keeps the pod alive so the user can `oc rsh` + # in, install nemo2riva, run LoRA training, or stage `.riva` + # model files for the riva-stt deployment to consume. + args: ["sleep infinity"] + env: + - name: HOME + value: /tmp + # Shared HF cache with the serving PVC so we don't re-download + # base checkpoints already pulled for other deployments. + - name: HF_HOME + value: /models/hf_home + - name: TRANSFORMERS_CACHE + value: /models/hf_home + - name: NEMO_CACHE_DIR + value: /models/nemo_cache + - name: NUMBA_CACHE_DIR + value: /tmp/numba_cache + - name: MPLCONFIGDIR + value: /tmp/matplotlib + resources: + # Training is memory-heavier than serving: DataLoader workers, + # dataset shards in RAM, checkpoint staging. Serving pods in + # this repo use 16-32Gi; this pod doubles it. + requests: + cpu: "4" + memory: 32Gi + nvidia.com/gpu: "1" + limits: + cpu: "16" + memory: 64Gi + nvidia.com/gpu: "1" + volumeMounts: + # Shared model cache: lets this pod drop the `.riva` output file + # where the riva-stt deployment will find it (/models/riva/*). + - name: model-cache + mountPath: /models + # Training-only PVC: dataset shards, LoRA checkpoints, logs. + - name: training-data + mountPath: /data + # /dev/shm for DataLoader workers — PyTorch spawns workers that + # share tensors via shared memory; the default 64MB is too small. + - name: shm + mountPath: /dev/shm + volumes: + - name: model-cache + persistentVolumeClaim: + claimName: vllm-model-cache + - name: training-data + persistentVolumeClaim: + claimName: nemo-training-data + - name: shm + emptyDir: + medium: Memory + sizeLimit: 8Gi +# No Service, no Route: this deployment is accessed interactively +# through `oc rsh` only. No network endpoint is exposed. From d6fff3947634c97bac36bdf5c48cd5224b0e4bf5 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 6 May 2026 15:22:15 -0400 Subject: [PATCH 2/2] k8s(nemo-training): propagate standard labels to pod template Codex review flagged that the app.kubernetes.io/part-of and /component labels were only set on the Deployment object. Deployment.metadata.labels do not propagate to Pods, so selectors and observability tooling looking for those labels on pods would not match. Duplicate the labels under spec.template.metadata.labels so they are present on both the Deployment object and the Pods it creates. Addresses Codex review feedback on PR #15. Co-Authored-By: Claude Opus 4.7 (1M context) --- k8s/nemo-training-deployment.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/k8s/nemo-training-deployment.yaml b/k8s/nemo-training-deployment.yaml index b3c0891..555ef49 100644 --- a/k8s/nemo-training-deployment.yaml +++ b/k8s/nemo-training-deployment.yaml @@ -20,8 +20,13 @@ spec: app: nemo-training template: metadata: + # Pod-level labels must duplicate the Deployment's standard labels + # (Deployment.metadata.labels do not propagate to Pods). Selectors + # and observability tooling match on Pod labels. labels: app: nemo-training + app.kubernetes.io/part-of: accent-stt + app.kubernetes.io/component: training spec: # H100 taint, matching every existing deployment in this repo. tolerations: