03-torchrun-multi-node-basic-tools.json

{
  "_comment": "Multi-Node (2 nodes, 2 GPUs each) - Basic Configuration",
  "_description": "Configuration for distributed workload across 2 nodes with 2 GPUs per node (4 GPUs total)",
  "_use_case": "Multi-node distributed execution testing on busy clusters",
  
  "gpu_vendor": "AMD",
  "guest_os": "UBUNTU",

  "tools": [{"name": "gpu_info_power_profiler"}, {"name": "gpu_info_vram_profiler"}, {"name": "rocprof"}, {"name": "rpd"}, {"name": "miopen_trace"}, {"name": "rocblas_trace"}, {"name": "tensile_trace"}],
  
  "k8s": {
    "kubeconfig": "~/.kube/config",
    "namespace": "default",
    "gpu_count": 2,
    
    "memory": "64Gi",
    "memory_limit": "128Gi",
    "cpu": "16",
    "cpu_limit": "32",
    
    "image_pull_policy": "Always",
    "backoff_limit": 3,
    "host_ipc": true
  },
  
  "distributed": {
    "enabled": true,
    "backend": "nccl",
    "launcher": "torchrun",
    "nnodes": 2,
    "nproc_per_node": 2,
    "master_port": 29500
  },
  
  "env_vars": {
    "NCCL_DEBUG": "WARN",
    "NCCL_DEBUG_SUBSYS": "INIT,NET",
    "NCCL_IB_DISABLE": "1",
    "NCCL_SOCKET_IFNAME": "eth0",
    "TORCH_NCCL_HIGH_PRIORITY": "1",
    "GPU_MAX_HW_QUEUES": "2",
    "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1",
    "NCCL_TIMEOUT": "600",
    "HSA_ENABLE_SDMA": "0",
    "OMP_NUM_THREADS": "8",
    "MIOPEN_FIND_MODE": "1",
    "MIOPEN_USER_DB_PATH": "/tmp/.miopen",
    "HSA_FORCE_FINE_GRAIN_PCIE": "1",
    "HSA_NO_SCRATCH_RECLAIM": "1",
    "RCCL_ENABLE_HIPGRAPH": "0"
  },
  
  "_env_var_notes": {
    "NCCL_DEBUG": "Changed to WARN for cleaner logs (use INFO for debugging)",
    "MIOPEN_FIND_MODE": "1 = Use compiled kernels, avoid find-db warnings",
    "MIOPEN_USER_DB_PATH": "Writable location for MIOpen cache",
    "HSA_FORCE_FINE_GRAIN_PCIE": "Helps with IOMMU-related warnings",
    "RCCL_ENABLE_HIPGRAPH": "Disable for compatibility",
    "NCCL_MIN_NCHANNELS": "Removed (warning says ignored for <8 GPUs)"
  },
  
  "debug": false
}