Traceback (most recent call last):
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 816, in wrapper
    self._join_processes(fn)
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 1086, in _join_processes
    self._check_return_codes(fn, elapsed_time)
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 1126, in _check_return_codes
    raise RuntimeError(error)
RuntimeError: Process 2 exited with error code 10 and exception:
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 971, in run_test
    getattr(self, test_name)()
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 818, in wrapper
    fn()
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_utils.py", line 3444, in wrapper
    method(*args, **kwargs)
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 235, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/distributed/distributed_test.py", line 4893, in test_ddp_apply_optim_in_backward
    self._test_ddp_apply_optim_in_backward(
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/distributed/distributed_test.py", line 4875, in _test_ddp_apply_optim_in_backward
    self.assertEqual(
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_utils.py", line 4441, in assertEqual
    raise error_metas.pop()[0].to_error(  # type: ignore[index]
AssertionError: Tensor-likes are not close!

Mismatched elements: 15 / 3072 (0.5%)
Greatest absolute difference: 0.00732421875 at index (902, 1) (up to 1e-05 allowed)
Greatest relative difference: 8.4557705122279e-06 at index (185, 1) (up to 1.3e-06 allowed)
Params not equal at iteration 4

To execute this test, run the following from the base repo dir:
    python test/distributed/test_distributed_spawn.py TestDistBackendWithSpawn.test_ddp_apply_optim_in_backward

This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0

/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/backends/cudnn/__init__.py:175: UserWarning: cuDNN Benchmark limit is not supported in MIOpen and will have no effect. (Triggered internally at /var/lib/jenkins/pytorch/torch/csrc/cuda/Module.cpp:1990.)
  torch._C._cuda_set_cudnn_benchmark_limit(_benchmark_limit)

Process 2 terminated with exit code 10, terminating remaining processes.

Traceback (most recent call last):
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 816, in wrapper
    self._join_processes(fn)
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 1086, in _join_processes
    self._check_return_codes(fn, elapsed_time)
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 1126, in _check_return_codes
    raise RuntimeError(error)
RuntimeError: Process 1 exited with error code 10 and exception:
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 971, in run_test
    getattr(self, test_name)()
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 818, in wrapper
    fn()
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_utils.py", line 3444, in wrapper
    method(*args, **kwargs)
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.py", line 235, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/distributed/distributed_test.py", line 4893, in test_ddp_apply_optim_in_backward
    self._test_ddp_apply_optim_in_backward(
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/distributed/distributed_test.py", line 4875, in _test_ddp_apply_optim_in_backward
    self.assertEqual(
  File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_utils.py", line 4441, in assertEqual
    raise error_metas.pop()[0].to_error(  # type: ignore[index]
AssertionError: Tensor-likes are not close!

Mismatched elements: 9 / 3072 (0.3%)
Greatest absolute difference: 0.006103515625 at index (525, 1) (up to 1e-05 allowed)
Greatest relative difference: 3.6533190268528415e-06 at index (169, 0) (up to 1.3e-06 allowed)
Params not equal at iteration 4

To execute this test, run the following from the base repo dir:
    python test/distributed/test_distributed_spawn.py TestDistBackendWithSpawn.test_ddp_apply_optim_in_backward

This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0

/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/backends/cudnn/__init__.py:175: UserWarning: cuDNN Benchmark limit is not supported in MIOpen and will have no effect. (Triggered internally at /var/lib/jenkins/pytorch/torch/csrc/cuda/Module.cpp:1990.)
  torch._C._cuda_set_cudnn_benchmark_limit(_benchmark_limit)

Process 1 terminated with exit code 10, terminating remaining processes.