Debugging macos CI failures

.circleci/config.yml

@@ -32,20 +32,26 @@ commands:
     steps:
       - checkout
       - restore_cache:
-          key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ .Branch }}-<< parameters.py_version >>
+          key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >>
       - run:
           name: Preparing environment - Conda
           command: |
+            set -x
             if [[ -f ~/miniconda3/LICENSE.txt ]] ; then
               echo "miniconda installed already."
             else
-              curl -o Miniconda3-py38_4.8.3-MacOSX-x86_64.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.8.3-MacOSX-x86_64.sh
-              bash ./Miniconda3-py38_4.8.3-MacOSX-x86_64.sh -b
+              curl -o Miniconda3-latest-MacOSX-arm64.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh
+              uname -m
+              bash ./Miniconda3-latest-MacOSX-arm64.sh -b
             fi
             ~/miniconda3/bin/conda init bash
       - run:
           name: Preparing environment - Brew
           command: |
+            set -x
+            brew --version
+            brew untap Homebrew/homebrew-cask-versions
+            rm -rf $(brew --caskroom)/adoptopenjdk11  # https://github.com/Homebrew/homebrew-cask/issues/173201#issuecomment-2101662209
             brew update
             brew upgrade git
             brew install fish
@@ -57,7 +63,7 @@ commands:
             conda create -n hydra python=<< parameters.py_version >> -yqc conda-forge
             conda run -n hydra pip install nox --progress-bar off
       - save_cache:
-          key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ .Branch }}-<< parameters.py_version >>
+          key: -<< pipeline.parameters.cache_key_version >>-macos-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >>
           paths:
             - ~/miniconda3
             - ~/Library/Caches/Homebrew
@@ -95,12 +101,13 @@ commands:
     steps:
       - checkout
       - restore_cache:
-          key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ .Branch }}-<< parameters.py_version >>
+          key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >>
       - run:
           name: Preparing environment - system
+          no_output_timeout: 20m
           command: |
             choco install -y --no-progress miniconda3
-            choco install -y --no-progress openssl openjdk11jre
+            choco install -y --no-progress openjdk
             C:\tools\miniconda3\Scripts\conda.exe init powershell
       - run:
           name: Preparing environment - Hydra
@@ -109,7 +116,7 @@ commands:
             conda activate hydra
             pip install nox dataclasses --progress-bar off
       - save_cache:
-          key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ .Branch }}-<< parameters.py_version >>
+          key: -<< pipeline.parameters.cache_key_version >>-win-sys-{{ arch }}-{{ .Branch }}-<< parameters.py_version >>
           paths:
             - C:\tools\miniconda3
 
@@ -119,7 +126,8 @@ jobs:
       py_version:
         type: string
     macos:
-      xcode: "13.4.1"
+      xcode: "15.3.0"
+    resource_class: macos.m1.medium.gen1
     steps:
       - macos:
           py_version: << parameters.py_version >>
@@ -198,6 +206,7 @@ jobs:
         type: string
     macos:
       xcode: "13.4.1"
+    resource_class: macos.m1.medium.gen1
     steps:
       - macos:
           py_version: << parameters.py_version >>

noxfile.py

@@ -87,13 +87,13 @@ def print_installed_package_version(session: Session, package_name: str) -> None
     pip_list: str = session.run("pip", "list", silent=True)
     for line in pip_list.split("\n"):
         if package_name in line:
-            print(f"Installed {package_name} version: {line}")
+            logger.info(f"Installed {package_name} version: {line}")
 
 
 def install_hydra(session: Session, cmd: List[str]) -> None:
     # needed for build
     session.install("read-version", silent=SILENT)
-    # clean install hydra
+    # clean-install hydra
     session.chdir(BASE)
     if USE_OMEGACONF_DEV_VERSION:
         session.install("--pre", "omegaconf", silent=SILENT)

plugins/hydra_ray_launcher/integration_test_tools/create_integration_test_ami.py

@@ -24,7 +24,7 @@
 
 def _run_command(command: str) -> str:
     print(f"{str(datetime.now())} - Running: {command}")
-    output = subprocess.getoutput(command)
+    output = subprocess.getoutput(command)  # nosec B605
     print(f"{str(datetime.now())} - {output}")
     return output
 

plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py

@@ -16,7 +16,7 @@
 
 def _run_command(command: str) -> str:
     print(f"{str( datetime.now() )} - OUT: {command}")
-    output = subprocess.getoutput(command)
+    output = subprocess.getoutput(command)  # nosec B605
     print(f"{str( datetime.now() )} - OUT: {output}")
     return output
 

plugins/hydra_ray_launcher/tests/test_ray_aws_launcher.py

@@ -129,7 +129,7 @@
 
 def run_command(commands: str) -> str:
     log.info(f"running: {commands}")
-    output = subprocess.getoutput(commands)
+    output = subprocess.getoutput(commands)  # nosec B605
     log.info(f"outputs: {output}")
     return output
 

requirements/dev.txt

@@ -12,7 +12,6 @@ packaging
 pre-commit
 pytest
 pytest-benchmark
-pytest-snail
 read-version
 setuptools
 towncrier