diff --git a/src/post_training/slurm/job_llamafactory.sh.jinja b/src/post_training/slurm/job_llamafactory.sh.jinja index 088ba26..b241b8e 100644 --- a/src/post_training/slurm/job_llamafactory.sh.jinja +++ b/src/post_training/slurm/job_llamafactory.sh.jinja @@ -43,7 +43,7 @@ export WORLD_SIZE=$(( SLURM_NNODES * GPUS_PER_NODE )) # NCCL tuning for multi-node stability export NCCL_IB_TIMEOUT=120 -export NCCL_DEBUG=INFO +export NCCL_DEBUG=WARN export CUDA_DEVICE_MAX_CONNECTIONS=1 export OMP_NUM_THREADS=1 diff --git a/src/post_training/slurm/job_trl_container.sh.jinja b/src/post_training/slurm/job_trl_container.sh.jinja index 6269035..c963f3f 100644 --- a/src/post_training/slurm/job_trl_container.sh.jinja +++ b/src/post_training/slurm/job_trl_container.sh.jinja @@ -43,7 +43,7 @@ export WORLD_SIZE=$(( SLURM_NNODES * GPUS_PER_NODE )) # NCCL tuning for multi-node stability export NCCL_IB_TIMEOUT=120 -export NCCL_DEBUG=INFO +export NCCL_DEBUG=WARN # CUDA_DEVICE_MAX_CONNECTIONS=1 is a Megatron-LM tensor-parallel flag that # serializes CUDA streams and hurts ZeRO-2 overlap_comm — do not set it. export OMP_NUM_THREADS=1