diff --git a/.github/actions/defaults/action.yml b/.github/actions/defaults/action.yml new file mode 100644 index 00000000000..f45a75226c1 --- /dev/null +++ b/.github/actions/defaults/action.yml @@ -0,0 +1,21 @@ +name: Defaults +description: "Composite action that exposes default values for workflows (e.g., python_version)." + +inputs: + python-version: + description: "Default Python version to use in workflows" + required: false + default: '3.11' + +runs: + using: composite + steps: + - id: set + run: | + echo "python_version=${{ inputs.python-version }}" >> $GITHUB_OUTPUT + shell: bash + +outputs: + python_version: + description: "Python version string" + value: ${{ steps.set.outputs.python_version }} diff --git a/.github/actions/install-htcondor-pegasus/action.yml b/.github/actions/install-htcondor-pegasus/action.yml new file mode 100644 index 00000000000..42ae6265eb5 --- /dev/null +++ b/.github/actions/install-htcondor-pegasus/action.yml @@ -0,0 +1,24 @@ +name: Install HTCondor and Pegasus +description: "Composite action to install HTCondor and Pegasus on Ubuntu runners." +runs: + using: "composite" + steps: + - name: Install HTCondor + shell: bash + run: | + wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - + echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list + echo "deb-src http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list + sudo apt-get -o Acquire::Retries=3 update + sudo apt-get -o Acquire::Retries=3 install -y minihtcondor + # systemd may not be available in all runners, ignore failures when enabling + sudo systemctl start condor || true + sudo systemctl enable condor || true + + - name: Install Pegasus + shell: bash + run: | + wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - + echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu noble main" | sudo tee -a /etc/apt/sources.list + sudo apt-get -o Acquire::Retries=3 update + sudo apt-get -o Acquire::Retries=3 install -y pegasus=5.1.1-1+ubuntu24 diff --git a/.github/actions/install-system-deps/action.yml b/.github/actions/install-system-deps/action.yml new file mode 100644 index 00000000000..fedb526481d --- /dev/null +++ b/.github/actions/install-system-deps/action.yml @@ -0,0 +1,16 @@ +name: Install system packages +description: "Composite action to install common system packages (fftw3, intel-mkl, mpi, graphviz) on Ubuntu runners." + +runs: + using: 'composite' + steps: + - name: Update apt + shell: bash + run: | + sudo apt-get -o Acquire::Retries=3 update + + - name: Install system packages + shell: bash + run: | + set -euxo pipefail + sudo apt-get -o Acquire::Retries=3 install -y *fftw3* intel-mkl* mpi graphviz diff --git a/.github/actions/populate-caches/action.yml b/.github/actions/populate-caches/action.yml new file mode 100644 index 00000000000..c36b16851a7 --- /dev/null +++ b/.github/actions/populate-caches/action.yml @@ -0,0 +1,79 @@ +name: Populate caches (LAL aux + example GW data) +description: "Composite action that restores LAL auxiliary data and example GW data caches and downloads missing files." + +runs: + using: "composite" + steps: + - name: Restore LAL auxiliary data cache + id: restore-lal-aux + uses: actions/cache@v4 + with: + key: lal-aux-data + path: $HOME/lal_aux_data + + - name: Download LAL auxiliary data files on miss + if: ${{ steps.restore-lal-aux.outputs.cache-hit != 'true' }} + run: | + mkdir -p $HOME/lal_aux_data + pushd $HOME/lal_aux_data + curl --show-error --silent --fail --retry 3 --retry-delay 5 --retry-connrefused \ + --remote-name https://zenodo.org/records/14999310/files/SEOBNRv4ROM_v2.0.hdf5 \ + --remote-name https://zenodo.org/records/14999310/files/SEOBNRv4ROM_v3.0.hdf5 + popd + + - name: Restore example GW data cache + name: Populate caches (LAL aux + example GW data) + description: "Composite action that restores LAL auxiliary data and example GW data caches and downloads missing files." + + runs: + using: "composite" + steps: + - name: Restore LAL auxiliary data cache + id: restore-lal-aux + uses: actions/cache@v4 + with: + key: lal-aux-data + path: $HOME/lal_aux_data + + - name: Download LAL auxiliary data files on miss + if: ${{ steps.restore-lal-aux.outputs.cache-hit != 'true' }} + run: | + mkdir -p $HOME/lal_aux_data + pushd $HOME/lal_aux_data + curl --show-error --silent --fail --retry 3 --retry-delay 5 --retry-connrefused \ + --remote-name https://zenodo.org/records/14999310/files/SEOBNRv4ROM_v2.0.hdf5 \ + --remote-name https://zenodo.org/records/14999310/files/SEOBNRv4ROM_v3.0.hdf5 + popd + + - name: Restore example GW data cache + id: restore-example-gw + uses: actions/cache@v4 + with: + key: example-gw-data + path: | + docs/_include/*_TDI_v2.gwf + docs/_include/*_GWOSC_4KHZ_R1-1126257415-4096.gwf + docs/_include/*_LOSC_CLN_4_V1-1187007040-2048.gwf + examples/inference/lisa_smbhb_ldc/*_psd.txt + examples/inference/lisa_smbhb_ldc/*_TDI_v2.gwf + examples/inference/lisa_smbhb_ldc/MBHB_params_v2_LISA_frame.pkl + examples/inference/margtime/*.gwf + examples/inference/multisignal/*.gwf + examples/inference/relative/*.gwf + examples/inference/relmarg/*.gwf + examples/inference/single/*.gwf + examples/search/*.gwf + + - name: Set path outputs for cached data + id: set_paths + run: | + echo "lal_data_path=$HOME/lal_aux_data" >> $GITHUB_OUTPUT + echo "example_gw_cache_key=example-gw-data" >> $GITHUB_OUTPUT + + outputs: + lal_data_path: + description: "Path where LAL auxiliary data is stored on the runner" + value: ${{ steps.set_paths.outputs.lal_data_path }} + example_gw_cache_key: + description: "Cache key used for example GW data" + value: ${{ steps.set_paths.outputs.example_gw_cache_key }} diff --git a/.github/workflows/bank-compress-workflow.yml b/.github/workflows/bank-compress-workflow.yml index d97b22ed798..e21cbbb3569 100644 --- a/.github/workflows/bank-compress-workflow.yml +++ b/.github/workflows/bank-compress-workflow.yml @@ -12,26 +12,17 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v1 + - name: Load defaults + id: defaults + uses: ./.github/actions/defaults - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - - name: install condor - run: | - wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - - echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - echo "deb-src http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install minihtcondor - sudo systemctl start condor - sudo systemctl enable condor - - name: install pegasus - run: | - wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu noble main" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.1.1-1+ubuntu24 - - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* + python-version: ${{ steps.defaults.outputs.python_version }} + - name: Install condor & pegasus + uses: ./.github/actions/install-htcondor-pegasus + - name: Install system dependencies + uses: ./.github/actions/install-system-deps - name: Install pycbc run: | python -m pip install --upgrade pip setuptools diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 79294148ab8..8761ab93dc6 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -13,67 +13,37 @@ jobs: max-parallel: 60 matrix: os: [ubuntu-24.04] - python-version: ['3.11', '3.12', '3.13'] + python_version: ['3.11', '3.12', '3.13'] test-type: [unittest, search, docs] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Populate caches (LAL aux + example GW data) + id: populate + uses: ./.github/actions/populate-caches + - name: Expose LAL_DATA_PATH to subsequent steps + run: echo "LAL_DATA_PATH=${{ steps.populate.outputs.lal_data_path }}" >> $GITHUB_ENV + - name: Set up Python ${{ matrix.python_version }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python_version }} - name: installing system packages + uses: .github/actions/install-system-deps + - name: Install python dependencies run: | - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install *fftw3* mpi intel-mkl* graphviz pip install tox pip setuptools --upgrade - - name: Cache LAL auxiliary data files - id: cache-lal-aux-data - uses: actions/cache@v4 - with: - key: lal-aux-data - path: ~/lal_aux_data - - if: ${{ steps.cache-lal-aux-data.outputs.cache-hit != 'true' }} - name: Download LAL auxiliary data files - run: | - mkdir ~/lal_aux_data - pushd ~/lal_aux_data - curl --show-error --silent \ - --remote-name https://zenodo.org/records/14999310/files/SEOBNRv4ROM_v2.0.hdf5 \ - --remote-name https://zenodo.org/records/14999310/files/SEOBNRv4ROM_v3.0.hdf5 - popd - - name: Cache example GW data - id: cache-example-gw-data - uses: actions/cache@v4 - with: - key: example-gw-data - path: | - docs/_include/*_TDI_v2.gwf - docs/_include/*_GWOSC_4KHZ_R1-1126257415-4096.gwf - docs/_include/*_LOSC_CLN_4_V1-1187007040-2048.gwf - examples/inference/lisa_smbhb_ldc/*_psd.txt - examples/inference/lisa_smbhb_ldc/*_TDI_v2.gwf - examples/inference/lisa_smbhb_ldc/MBHB_params_v2_LISA_frame.pkl - examples/inference/margtime/*.gwf - examples/inference/multisignal/*.gwf - examples/inference/relative/*.gwf - examples/inference/relmarg/*.gwf - examples/inference/single/*.gwf - name: run pycbc test suite run: | - export LAL_DATA_PATH=$HOME/lal_aux_data tox -e py-${{matrix.test-type}} - name: check help messages work if: matrix.test-type == 'unittest' run: | - export LAL_DATA_PATH=$HOME/lal_aux_data tox -e py-help - name: run inference tests if: matrix.test-type == 'search' run: | - export LAL_DATA_PATH=$HOME/lal_aux_data tox -e py-inference - name: store documentation page - if: matrix.test-type == 'docs' && matrix.python-version == '3.12' + if: matrix.test-type == 'docs' && matrix.python_version == '3.12' uses: actions/upload-artifact@v4 with: name: documentation-page diff --git a/.github/workflows/check_code.yml b/.github/workflows/check_code.yml index d0e8b2ffb9b..f8f8bb6172b 100644 --- a/.github/workflows/check_code.yml +++ b/.github/workflows/check_code.yml @@ -6,10 +6,13 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 + - name: Load defaults + id: defaults + uses: ./.github/actions/defaults - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: ${{ steps.defaults.outputs.python_version }} - name: Install flake8 run: pip install flake8 - name: Checking executables for unused imports diff --git a/.github/workflows/inference-workflow.yml b/.github/workflows/inference-workflow.yml index 0db7289855f..1539ac54800 100644 --- a/.github/workflows/inference-workflow.yml +++ b/.github/workflows/inference-workflow.yml @@ -7,26 +7,17 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v1 + - name: Load defaults + id: defaults + uses: ./.github/actions/defaults - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - - name: install condor - run: | - wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - - echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - echo "deb-src http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install minihtcondor - sudo systemctl start condor - sudo systemctl enable condor - - name: install pegasus - run: | - wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu noble main" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.1.1-1+ubuntu24 - - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* + python-version: ${{ steps.defaults.outputs.python_version }} + - name: install condor and pegasus + uses: ./.github/actions/install-htcondor-pegasus + - name: install system dependecies + uses: ./.github/actions/install-system-deps - name: Install pycbc run: | python -m pip install --upgrade pip setuptools diff --git a/.github/workflows/mac-test.yml b/.github/workflows/mac-test.yml index cbe7b6c0c8d..dd71eae29f0 100644 --- a/.github/workflows/mac-test.yml +++ b/.github/workflows/mac-test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 4 matrix: os: [macos-latest] - python-version: + python_version: - '3.11' - '3.12' - '3.13' @@ -32,8 +32,8 @@ jobs: # increment to reset cache CACHE_NUMBER: 0 with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ matrix.python-version}}-${{ env.CACHE_NUMBER }} + path: $HOME/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ matrix.python_version}}-${{ env.CACHE_NUMBER }} - name: Configure conda uses: conda-incubator/setup-miniconda@v3 @@ -41,7 +41,7 @@ jobs: activate-environment: test channels: conda-forge miniforge-version: latest - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python_version }} - name: Conda info run: conda info --all diff --git a/.github/workflows/search-workflow.yml b/.github/workflows/search-workflow.yml index f3771636c28..f337412d997 100644 --- a/.github/workflows/search-workflow.yml +++ b/.github/workflows/search-workflow.yml @@ -14,26 +14,17 @@ jobs: PEGASUS_METRICS: 'false' steps: - uses: actions/checkout@v1 + - name: Load defaults + id: defaults + uses: ./.github/actions/defaults - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - - name: install condor - run: | - wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - - echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - echo "deb-src http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install minihtcondor - sudo systemctl start condor - sudo systemctl enable condor - - name: install pegasus - run: | - wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu noble main" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.1.1-1+ubuntu24 - - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* + python-version: ${{ steps.defaults.outputs.python_version }} + - name: Install condor & pegasus + uses: ./.github/actions/install-htcondor-pegasus + - name: installing system packages + uses: .github/actions/install-system-deps - name: Install pycbc run: | python -m pip install --upgrade pip setuptools diff --git a/.github/workflows/tmpltbank-workflow.yml b/.github/workflows/tmpltbank-workflow.yml index 87b10f561d4..3ecedf52548 100644 --- a/.github/workflows/tmpltbank-workflow.yml +++ b/.github/workflows/tmpltbank-workflow.yml @@ -11,26 +11,17 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v1 + - name: Load defaults + id: defaults + uses: ./.github/actions/defaults - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - - name: install condor - run: | - wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - - echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - echo "deb-src http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install minihtcondor - sudo systemctl start condor - sudo systemctl enable condor - - name: install pegasus - run: | - wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu noble main" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.1.1-1+ubuntu24 - - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* + python-version: ${{ steps.defaults.outputs.python_version }} + - name: Install condor & pegasus + uses: ./.github/actions/install-htcondor-pegasus + - name: Install system dependencies + uses: ./.github/actions/install-system-deps - name: Install pycbc run: | python -m pip install --upgrade pip setuptools diff --git a/.github/workflows/tut-test.yml b/.github/workflows/tut-test.yml index 4f021152cf3..6973c83e87e 100644 --- a/.github/workflows/tut-test.yml +++ b/.github/workflows/tut-test.yml @@ -13,17 +13,17 @@ jobs: max-parallel: 60 matrix: os: [ubuntu-24.04] - python-version: ['3.11', '3.12', '3.13'] + python_version: ['3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python_version }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - name: installing packages + python-version: ${{ matrix.python_version }} + - name: Install system dependencies + uses: ./.github/actions/install-system-deps + - name: installing python packages run: | - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install *fftw3* mpi intel-mkl* pip install tox pip setuptools notebook --upgrade pip install . - name: retrieving pycbc tutorials diff --git a/.github/workflows/workflow-tests.yml b/.github/workflows/workflow-tests.yml index dde087ae20c..03c3198f6ef 100644 --- a/.github/workflows/workflow-tests.yml +++ b/.github/workflows/workflow-tests.yml @@ -16,26 +16,17 @@ jobs: test-type: [simple_subworkflow_data, multilevel_subworkflow_data] steps: - uses: actions/checkout@v1 + - name: Load defaults + id: defaults + uses: ./.github/actions/defaults - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - - name: install condor - run: | - wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - - echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - echo "deb-src http://research.cs.wisc.edu/htcondor/ubuntu/8.9/focal focal contrib" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install minihtcondor - sudo systemctl start condor - sudo systemctl enable condor - - name: install pegasus - run: | - wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add - - echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu noble main" | sudo tee -a /etc/apt/sources.list - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install pegasus=5.1.1-1+ubuntu24 - - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl* + python-version: ${{ steps.defaults.outputs.python_version }} + - name: Install condor & pegasus + uses: ./.github/actions/install-htcondor-pegasus + - name: Install system dependencies + uses: ./.github/actions/install-system-deps - name: Install pycbc run: | python -m pip install --upgrade pip setuptools diff --git a/examples/search/get.sh b/examples/search/get.sh index ec4bd98635a..5901b12bae5 100644 --- a/examples/search/get.sh +++ b/examples/search/get.sh @@ -1,6 +1,30 @@ #!/bin/bash set -e -wget -nv https://dcc.ligo.org/public/0146/P1700341/001/H-H1_LOSC_CLN_4_V1-1186740069-3584.gwf -wget -nv https://dcc.ligo.org/public/0146/P1700341/001/L-L1_LOSC_CLN_4_V1-1186740069-3584.gwf -wget -nv https://dcc.ligo.org/public/0146/P1700341/001/V-V1_LOSC_CLN_4_V1-1186739813-4096.gwf +# List of frame files used by the example +FILES=( + "H-H1_LOSC_CLN_4_V1-1186740069-3584.gwf" + "L-L1_LOSC_CLN_4_V1-1186740069-3584.gwf" + "V-V1_LOSC_CLN_4_V1-1186739813-4096.gwf" +) + +for f in "${FILES[@]}"; do + # If file is already in current working directory, skip + if [ -f "./$f" ]; then + echo "Found $f in working directory; skipping download." + continue + fi + + # If file is present in examples/search (i.e. the cache), then copy it + if [ -f "examples/search/$f" ]; then + echo "Found $f in examples/search (cache); copying." + cp "examples/search/$f" ./ + continue + fi + + # Otherwise, download from the DCC + echo "Downloading $f from DCC..." + wget -nv "https://dcc.ligo.org/public/0146/P1700341/001/$f" + # Copy into the cache folder + cp $f examples/search/ +done