Upgrade to CUDA 11.7 and remove V1 workflows (#1398)

Summary:
Define the default Python version in `python_utils.py`.
Define the default CUDA version in `cuda_utils.py`.

To upgrade CUDA version:
1. Change the default CUDA version in `utils/cuda_utils.py`
2. Change the default CUDA version in the Dockerfile.

Fixes #1390

Pull Request resolved: #1398

Reviewed By: atalman

Differential Revision: D43086532

Pulled By: xuzhao9

fbshipit-source-id: 04b99af060be7dec951ac23558cef01e99de4472

Author: xuzhao9

.github/workflows/pr-gha-runner.yml

@@ -4,10 +4,7 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHON_VERSION: "3.10"
-  CUDA_VERSION: "cu116"
   CONDA_ENV: "pr-test"
-  MAGMA_VERSION: "magma-cuda116"
   SETUP_INSTANCE_SCRIPT: "/workspace/setup_instance.sh"
 
 jobs:
@@ -26,19 +23,15 @@ jobs:
       - name: Setup Conda Env
         run: |
           . "${SETUP_INSTANCE_SCRIPT}"
-          conda create -n "${CONDA_ENV}" python="${PYTHON_VERSION}"
+          python utils/python_utils.py --create-conda-env "${CONDA_ENV}"
           conda activate "${CONDA_ENV}"
-          conda install -y "${MAGMA_VERSION}" -c pytorch
-          conda install -y numpy requests ninja pyyaml setuptools gitpython beautifulsoup4 regex
-          # weasyprint requires libglib from conda
-          # doctr_reco_predictor needs weasyprint, which needs libglib and pango
-          conda install -y expecttest libglib pango -c conda-forge
-          pip install unittest-xml-reporting
+          sudo python utils/cuda_utils.py --setup-cuda-softlink
+          python utils/cuda_utils.py --install-torch-deps
       - name: Install PyTorch nightly
         run: |
           . "${SETUP_INSTANCE_SCRIPT}"
           conda activate "${CONDA_ENV}"
-          pip install --pre torch torchvision torchtext torchaudio -f https://download.pytorch.org/whl/nightly/cu116/torch_nightly.html
+          python utils/cuda_utils.py --install-torch-nightly
       - name: Install TorchBench
         run: |
           . "${SETUP_INSTANCE_SCRIPT}"

.github/workflows/userbenchmark-a100.yml

@@ -9,12 +9,7 @@ on:
       userbenchmark_options:
         description: "Option of the user benchmark to run"
 env:
-  PYTHON_VERSION: "3.8"
-  TENSORRT_PYTHON_VERSION: "cp38"
-  TENSORRT_VERSION: "8.2.4.2"
-  CUDA_VERSION: "cu116"
   CONDA_ENV_NAME: "userbenchmarks-ci"
-  MAGMA_VERSION: "magma-cuda116"
   PLATFORM_NAME: "gcp_a100"
   TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN }}
   SETUP_SCRIPT: "/workspace/setup_instance.sh"
@@ -30,7 +25,8 @@ jobs:
       - name: Create conda environment
         run: |
           . "${SETUP_SCRIPT}"
-          conda create -y -q --name "${CONDA_ENV_NAME}" python="${PYTHON_VERSION}"
+          python benchmark/utils/python_utils.py --create-conda-env "${CONDA_ENV_NAME}"
+          sudo python utils/cuda_utils.py --setup-cuda-softlink
       - name: Tune Nvidia GPU
         run: |
           . "${SETUP_SCRIPT}"
@@ -41,11 +37,7 @@ jobs:
         run: |
           . "${SETUP_SCRIPT}" && conda activate "${CONDA_ENV_NAME}"
           pushd benchmark
-          # Install dependencies
-          conda install -y numpy requests ninja pyyaml setuptools gitpython beautifulsoup4 regex
-          conda install -y expecttest libglib pango -c conda-forge
-          conda install -y -c pytorch "${MAGMA_VERSION}"
-          pip install requests bs4 argparse gitpython boto3 pyyaml numpy regex psutil
+          python utils/cuda_utils.py --install-torch-deps
           # Check if nightly builds are available
           NIGHTLIES=$(python torchbenchmark/util/torch_nightly.py --packages torch)
           # If failed, the script will generate empty result
@@ -54,8 +46,7 @@ jobs:
               exit 1
           fi
           # Install PyTorch nightly and domain libraries from pip
-          pip install --pre torch torchtext torchvision torchaudio \
-            -f https://download.pytorch.org/whl/nightly/${CUDA_VERSION}/torch_nightly.html
+          python utils/cuda_utils.py --install-torch-nightly
           # make sure pytorch+cuda works
           python -c "import torch; torch.cuda.init()"
       - name: Install TorchBench

.github/workflows/userbenchmark-t4-metal.yml

@@ -9,10 +9,7 @@ on:
       userbenchmark_options:
         description: "Option of the user benchmark to run"
 env:
-  PYTHON_VERSION: "3.8"
-  CUDA_VERSION: "cu116"
   CONDA_ENV_NAME: "userbenchmarks-ci"
-  MAGMA_VERSION: "magma-cuda116"
   PLATFORM_NAME: "aws_t4_metal"
   TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN }}
   SETUP_SCRIPT: "/data/nvme/bin/setup_instance.sh"
@@ -27,19 +24,17 @@ jobs:
           path: benchmark
       - name: Create conda environment
         run: |
-          conda create -y -q --name "${CONDA_ENV_NAME}" python="${PYTHON_VERSION}"
+          python benchmark/utils/python_utils.py --create-conda-env "${CONDA_ENV_NAME}"
+          sudo python utils/cuda_utils.py --setup-cuda-softlink
       - name: Install PyTorch nightly
         run: |
           . "${SETUP_SCRIPT}" && . activate "${CONDA_ENV_NAME}"
           pushd benchmark
           # Install dependencies
-          conda install -y pyyaml numpy ninja regex gitpython setuptools beautifulsoup4 requests
-          conda install -y libglib pango expecttest -c conda-forge
+          python utils/cuda_utils.py --install-torch-deps
           # check the machine is tuned
-          pip install -U py-cpuinfo psutil distro
+          pip install -U py-cpuinfo psutil distro boto3
           sudo ${HOME}/miniconda3/envs/${CONDA_ENV_NAME}/bin/python3 torchbenchmark/util/machine_config.py
-          conda install -y -c pytorch "${MAGMA_VERSION}"
-          pip install requests bs4 argparse gitpython boto3 regex
           # Check if nightly builds are available
           NIGHTLIES=$(python torchbenchmark/util/torch_nightly.py --packages torch)
           # If failed, the script will generate empty result
@@ -48,8 +43,7 @@ jobs:
               exit 1
           fi
           # Install PyTorch and torchvision nightly from pip
-          pip install --pre torch torchvision torchtext torchaudio \
-            -f https://download.pytorch.org/whl/nightly/${CUDA_VERSION}/torch_nightly.html
+          python utils/cuda_utils.py --install-torch-nightly
           # make sure pytorch+cuda works
           python -c "import torch; torch.cuda.init()"
       - name: Install TorchBench