madengine/examples/k8s-configs/basic/04-torchrun-multi-node-advanced.json at develop · ROCm/madengine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
{
  "_comment": "Multi-Node (4 nodes, 2 GPUs each) - Advanced Configuration",
  "_description": "Full-featured configuration for large-scale distributed workloads with PVCs, tolerations, and node affinity",
  "_use_case": "Multi-node distributed execution with advanced features on busy clusters (8 GPUs total)",

  "gpu_vendor": "AMD",
  "guest_os": "UBUNTU",

  "k8s": {
    "kubeconfig": "~/.kube/config",
    "namespace": "ml-training",
    "gpu_count": 2,
    "gpu_resource_name": "amd.com/gpu",

    "memory": "128Gi",
    "memory_limit": "192Gi",
    "cpu": "24",
    "cpu_limit": "32",

    "image_pull_policy": "IfNotPresent",
    "backoff_limit": 5,
    "host_ipc": true,

    "node_selector": {
      "feature.node.kubernetes.io/amd-gpu-mi300x": "true",
      "topology.kubernetes.io/zone": "us-west-2a",
      "workload-type": "ml-training"
    },

    "tolerations": [
      {
        "key": "gpu",
        "operator": "Equal",
        "value": "amd",
        "effect": "NoSchedule"
      },
      {
        "key": "workload",
        "operator": "Equal",
        "value": "training",
        "effect": "NoSchedule"
      }
    ],

    "results_pvc": "ml-results-pvc",
    "data_pvc": "ml-datasets-pvc",

    "output_dir": "./k8s_manifests/multi-node"
  },

  "distributed": {
    "enabled": true,
    "backend": "nccl",
    "launcher": "torchrun",
    "nnodes": 4,
    "nproc_per_node": 2,
    "master_port": 29500
  },

  "env_vars": {
    "NCCL_DEBUG": "WARN",
    "NCCL_DEBUG_SUBSYS": "INIT,NET,GRAPH",
    "NCCL_IB_DISABLE": "1",
    "NCCL_SOCKET_IFNAME": "eth0",
    "TORCH_NCCL_HIGH_PRIORITY": "1",
    "GPU_MAX_HW_QUEUES": "2",
    "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1",
    "NCCL_TIMEOUT": "600",
    "HSA_ENABLE_SDMA": "0",
    "HSA_FORCE_FINE_GRAIN_PCIE": "1",
    "OMP_NUM_THREADS": "8",
    "MIOPEN_FIND_MODE": "1",
    "MIOPEN_USER_DB_PATH": "/tmp/.miopen",
    "RCCL_ENABLE_HIPGRAPH": "0"
  },

  "_env_var_notes": {
    "NCCL_DEBUG": "Changed to WARN for cleaner logs (use INFO for debugging)",
    "MIOPEN_FIND_MODE": "1 = Use compiled kernels, avoid find-db warnings",
    "MIOPEN_USER_DB_PATH": "Writable location for MIOpen cache",
    "HSA_FORCE_FINE_GRAIN_PCIE": "Helps with IOMMU-related warnings",
    "RCCL_ENABLE_HIPGRAPH": "Disable for compatibility",
    "NCCL_MIN_NCHANNELS": "Removed (warning says ignored for <8 GPUs)"
  },

  "debug": false
}