Replaced torch.distributed.launch with torchrun (#206)

* replaced torch.distributed.launch with torchrun

* fixed code for torch.distributed.launch

* changed torch.distributed.run to torchrun

* reverted some unnecessary changes

* reverted some more changes

* updated scripts/run_tests.sh

Author: sayantan1410

scripts/run_tests.sh

@@ -49,8 +49,8 @@ run_launch() {
   for dir in $(find ./dist-tests/$1-launch -type d)
   do
     cd $dir
-    python -m torch.distributed.launch \
-      --nproc_per_node 2 --use_env \
+    torchrun \
+      --nproc_per_node 2 \
       main.py --backend gloo --data_path ~/data \
       --train_batch_size 2 \
       --eval_batch_size 2 \

src/templates/template-common/README.md

@@ -3,42 +3,42 @@
 #::: if (it.nproc_per_node) { :::#
 #::: if (it.nnodes > 1 && it.master_addr && it.master_port) { :::#
 
-### Multi Node, Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi Node, Multi GPU Training (`torchrun`) (recommended)
 
 - Execute on master node
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank 0 \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 - Execute on worker nodes
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank <node_rank> \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 #::: } else { :::#
 
-### Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi GPU Training (`torchrun`) (recommended)
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 

src/templates/template-text-classification/README.md

@@ -19,42 +19,42 @@ pip install -r requirements.txt --progress-bar off -U
 #::: if (it.nproc_per_node) { :::#
 #::: if (it.nnodes > 1 && it.master_addr && it.master_port) { :::#
 
-### Multi Node, Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi Node, Multi GPU Training (`torchrun`) (recommended)
 
 - Execute on master node
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank 0 \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 - Execute on worker nodes
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank <node_rank> \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 #::: } else { :::#
 
-### Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi GPU Training (`torchrun`) (recommended)
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 

src/templates/template-vision-classification/README.md

@@ -19,42 +19,42 @@ pip install -r requirements.txt --progress-bar off -U
 #::: if (it.nproc_per_node) { :::#
 #::: if (it.nnodes > 1 && it.master_addr && it.master_port) { :::#
 
-### Multi Node, Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi Node, Multi GPU Training (`torchrun`) (recommended)
 
 - Execute on master node
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank 0 \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 - Execute on worker nodes
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank <node_rank> \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 #::: } else { :::#
 
-### Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi GPU Training (`torchrun`) (recommended)
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 

src/templates/template-vision-dcgan/README.md

@@ -19,42 +19,42 @@ pip install -r requirements.txt --progress-bar off -U
 #::: if (it.nproc_per_node) { :::#
 #::: if (it.nnodes > 1 && it.master_addr && it.master_port) { :::#
 
-### Multi Node, Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi Node, Multi GPU Training (`torchrun`) (recommended)
 
 - Execute on master node
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank 0 \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 - Execute on worker nodes
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank <node_rank> \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 #::: } else { :::#
 
-### Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi GPU Training (`torchrun`) (recommended)
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 

src/templates/template-vision-segmentation/README.md

@@ -19,42 +19,42 @@ pip install -r requirements.txt --progress-bar off -U
 #::: if (it.nproc_per_node) { :::#
 #::: if (it.nnodes > 1 && it.master_addr && it.master_port) { :::#
 
-### Multi Node, Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi Node, Multi GPU Training (`torchrun`) (recommended)
 
 - Execute on master node
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank 0 \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 - Execute on worker nodes
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
   --nnodes #:::= it.nnodes :::# \
   --node_rank <node_rank> \
   --master_addr #:::= it.master_addr :::# \
   --master_port #:::= it.master_port :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```
 
 #::: } else { :::#
 
-### Multi GPU Training (`torch.distributed.launch`) (recommended)
+### Multi GPU Training (`torchrun`) (recommended)
 
 ```sh
-python -m torch.distributed.launch \
+torchrun \
   --nproc_per_node #:::= it.nproc_per_node :::# \
-  --use_env main.py \
+  main.py \
   --backend #:::= it.backend :::#
 ```