repro profiler bug on abort

tushar00jain · tushar00jain · commit b2f862d88f54 · 2025-10-22T10:51:15.000-07:00
diff --git a/run_ft.sh b/run_ft.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC=0 TORCH_NCCL_DUMP_ON_TIMEOUT=0 TORCH_NCCL_TRACE_BUFFER_SIZE=0 TORCH_SHARE_RDZV_TCP_STORE=1 LOGLEVEL=INFO NCCL_DEBUG_SUBSYS=ALL NCCL_DEBUG=INFO TORCH_CPP_LOG_LEVEL=INFO CUDA_LAUNCH_BLOCKING=1 CUDA_VISIBLE_DEVICES=0,1 NGPU=2 ./run_train.sh --training.local_batch_size=2 --parallelism.data_parallel_shard_degree=2 --profiling.enable_profiling --profiling.profile_freq=1 --profiling.profiler_active=1 --profiling.profiler_warmup=0 --training.steps=1000 --comm.train_timeout_seconds=1 --comm.trace_buf_size=0
diff --git a/torchtitan/train.py b/torchtitan/train.py
@@ -661,7 +661,7 @@ def sigabrt_handler(signal):
                 if self.step == 1:
                     dist_utils.set_pg_timeouts(
                         timeout=timedelta(
-                            seconds=job_config.comm.train_timeout_seconds
+                            milliseconds=job_config.comm.train_timeout_seconds
                         ),
                         world_mesh=self.parallel_dims.world_mesh,
                     )

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#!/bin/bash`
	`2`	`+`
	`3`	+TORCH_NCCL_WAIT_TIMEOUT_DUMP_MILSEC=0 TORCH_NCCL_DUMP_ON_TIMEOUT=0 TORCH_NCCL_TRACE_BUFFER_SIZE=0 TORCH_SHARE_RDZV_TCP_STORE=1 LOGLEVEL=INFO NCCL_DEBUG_SUBSYS=ALL NCCL_DEBUG=INFO TORCH_CPP_LOG_LEVEL=INFO CUDA_LAUNCH_BLOCKING=1 CUDA_VISIBLE_DEVICES=0,1 NGPU=2 ./run_train.sh --training.local_batch_size=2 --parallelism.data_parallel_shard_degree=2 --profiling.enable_profiling --profiling.profile_freq=1 --profiling.profiler_active=1 --profiling.profiler_warmup=0 --training.steps=1000 --comm.train_timeout_seconds=1 --comm.trace_buf_size=0
Original file line number	Diff line number	Diff line change
`@@ -661,7 +661,7 @@ def sigabrt_handler(signal):`
`661`	`661`	`if self.step == 1:`
`662`	`662`	`dist_utils.set_pg_timeouts(`
`663`	`663`	`timeout=timedelta(`
`664`		`- seconds=job_config.comm.train_timeout_seconds`
	`664`	`+ milliseconds=job_config.comm.train_timeout_seconds`
`665`	`665`	`),`
`666`	`666`	`world_mesh=self.parallel_dims.world_mesh,`
`667`	`667`	`)`