madengine/examples/k8s-configs/basic/03-torchrun-multi-node-basic.json at develop · ROCm/madengine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
{
  "_comment": "Multi-Node (2 nodes, 2 GPUs each) - Basic Configuration",
  "_description": "Configuration for distributed workload across 2 nodes with 2 GPUs per node (4 GPUs total)",
  "_use_case": "Multi-node distributed execution testing on busy clusters",

  "gpu_vendor": "AMD",
  "guest_os": "UBUNTU",

  "k8s": {
    "kubeconfig": "~/.kube/config",
    "namespace": "default",
    "gpu_count": 2,

    "memory": "64Gi",
    "memory_limit": "128Gi",
    "cpu": "16",
    "cpu_limit": "32",

    "image_pull_policy": "Always",
    "backoff_limit": 3,
    "host_ipc": true,
    "recreate_shared_data_pvc": true
  },

  "distributed": {
    "enabled": true,
    "backend": "nccl",
    "launcher": "torchrun",
    "nnodes": 2,
    "nproc_per_node": 2,
    "master_port": 29500
  },

  "env_vars": {
    "NCCL_DEBUG": "WARN",
    "NCCL_DEBUG_SUBSYS": "INIT,NET",
    "NCCL_IB_DISABLE": "1",
    "NCCL_SOCKET_IFNAME": "eth0",
    "TORCH_NCCL_HIGH_PRIORITY": "1",
    "GPU_MAX_HW_QUEUES": "2",
    "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1",
    "NCCL_TIMEOUT": "600",
    "HSA_ENABLE_SDMA": "0",
    "OMP_NUM_THREADS": "8",
    "MIOPEN_FIND_MODE": "1",
    "MIOPEN_USER_DB_PATH": "/tmp/.miopen",
    "HSA_FORCE_FINE_GRAIN_PCIE": "1",
    "HSA_NO_SCRATCH_RECLAIM": "1",
    "RCCL_ENABLE_HIPGRAPH": "0"
  },

  "_env_var_notes": {
    "NCCL_DEBUG": "Changed to WARN for cleaner logs (use INFO for debugging)",
    "MIOPEN_FIND_MODE": "1 = Use compiled kernels, avoid find-db warnings",
    "MIOPEN_USER_DB_PATH": "Writable location for MIOpen cache",
    "HSA_FORCE_FINE_GRAIN_PCIE": "Helps with IOMMU-related warnings",
    "RCCL_ENABLE_HIPGRAPH": "Disable for compatibility",
    "NCCL_MIN_NCHANNELS": "Removed (warning says ignored for <8 GPUs)"
  },

  "debug": false
}