Propagate failure/preemptation and set timeout for multinode training (…

…#278) * Add parameters in multinode training to propagate failure/preemptation and set timeout.
allenai · Aug 20, 2024 · 5834277 · 5834277
1 parent a9c76a4
commit 5834277
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 0 deletions.
diff --git a/configs/beaker_configs/default_finetune_lora_multinode.yaml b/configs/beaker_configs/default_finetune_lora_multinode.yaml
@@ -5,6 +5,9 @@ tasks:
     replicas: 4
     leaderSelection: true
     hostNetworking: true
+    propagateFailure: true
+    propagatePreemption: true
+    synchronizedStartTimeout: 15m
     image:
       beaker: Yizhongw03/open-instruct-multi-node
     command: [

diff --git a/configs/beaker_configs/default_finetune_multinode.yaml b/configs/beaker_configs/default_finetune_multinode.yaml
@@ -6,6 +6,9 @@ tasks:
     replicas: 4
     leaderSelection: true
     hostNetworking: true
+    propagateFailure: true
+    propagatePreemption: true
+    synchronizedStartTimeout: 15m
     image:
       beaker: nathanl/open_instruct_auto
     command: [

diff --git a/configs/beaker_configs/default_finetune_qlora_multinode.yaml b/configs/beaker_configs/default_finetune_qlora_multinode.yaml
@@ -5,6 +5,9 @@ tasks:
     replicas: 4
     leaderSelection: true
     hostNetworking: true
+    propagateFailure: true
+    propagatePreemption: true
+    synchronizedStartTimeout: 15m
     image:
       beaker: Yizhongw03/open-instruct-multi-node
     command: [