diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..b07361e52 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,5 @@ +coverage: + status: + project: + default: + threshold: 0.1% diff --git a/.github/workflows/basic-install.yml b/.github/workflows/basic-install.yml index a71bfc18e..1919042cc 100644 --- a/.github/workflows/basic-install.yml +++ b/.github/workflows/basic-install.yml @@ -22,7 +22,7 @@ jobs: matrix: # disable windows build test as bilby_cython is currently broken there os: [ubuntu-latest, macos-latest] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 1085b8d90..0ba8a082c 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -5,56 +5,60 @@ on: - cron: "15 0 * * *" workflow_dispatch: -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - jobs: build-container: permissions: - contents: read - packages: write attestations: write + contents: read id-token: write + packages: write runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["310", "311", "312"] + python-version: ["11", "12", "13"] + env: + LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }} steps: - uses: actions/checkout@v4 - - name: Setup dockerfile + + - name: Remove unnecessary files run: | - cp containers/v3-dockerfile-test-suite-python${{ matrix.python-version }} Dockerfile - cp containers/env-template.yml . + df . -h + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + df . -h - name: Login to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: - registry: ${{ env.REGISTRY }} + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - tags: type=raw,value=latest - images: ghcr.io/${{ env.IMAGE_NAME }}-python${{ matrix.python-version }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build and push Docker image id: push - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + uses: docker/build-push-action@v6 with: context: . - tags: ${{ steps.meta.outputs.tags }} + build-args: | + python_minor_version=${{ matrix.python-version }} + ENV_FILE=containers/environment.yml push: true - labels: ${{ steps.meta.outputs.labels }} - - # FIXME: disabled as it is not working well but should probably be added in later - # https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations/using-artifact-attestations-to-establish-provenance-for-builds - # - name: Generate artifact attestation - # uses: actions/attest-build-provenance@v1 - # with: - # subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} - # subject-digest: ${{ steps.push.outputs.digest }} - # push-to-registry: true + file: containers/Dockerfile + tags: ${{ env.LABEL }}:latest + cache-from: type=registry,ref=${{ env.LABEL }}:buildcache + cache-to: type=registry,ref=${{ env.LABEL }}:buildcache,mode=max + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v3 + with: + subject-name: ${{ env.LABEL }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index eeca5692d..a8949b39a 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -23,7 +23,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest - container: ghcr.io/bilby-dev/bilby-python311:latest + container: ghcr.io/bilby-dev/bilby-python312:latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 030526bc5..674a4cd68 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -40,6 +40,9 @@ jobs: - name: Python 3.12 version: 3.12 short-version: 312 + - name: Python 3.13 + version: 3.13 + short-version: 313 steps: - uses: actions/checkout@v4 diff --git a/containers/Dockerfile b/containers/Dockerfile new file mode 100644 index 000000000..21935d2e0 --- /dev/null +++ b/containers/Dockerfile @@ -0,0 +1,52 @@ +############################# +######## Setup base ######### +############################# + +FROM containers.ligo.org/docker/base:conda AS os-deps +LABEL name="bilby CI testing" \ +maintainer="Gregory Ashton , Colm Talbot " + +############################# +##### Download ROQ data ##### +############################# + +FROM debian:bookworm-slim AS data +WORKDIR /roq_basis +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget \ + && rm -rf /var/lib/apt/lists/* +# Add the ROQ data to the image +RUN wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \ + && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \ + && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \ + && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \ + && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \ + && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_addcal.hdf5 \ + && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_multiband_addcal.hdf5 + +############################# +##### Build environment ##### +############################# + +FROM os-deps AS build-env +ARG ENV_FILE=environment.yml +COPY ${ENV_FILE} env.yml +ARG python_major_version=3 +ARG python_minor_version=12 + +RUN echo " - python=${python_major_version}.${python_minor_version}" >> env.yml +ENV conda_env=python${python_major_version}${python_minor_version} + +RUN mamba env create -f env.yml -n ${conda_env} +RUN echo "source activate ${conda_env}" > ~/.bashrc +ENV PATH=/opt/conda/envs/${conda_env}/bin:$PATH +RUN /bin/bash -c "source activate ${conda_env}" +RUN mamba clean --all + +############################# +### Finalize environment #### +############################# + +FROM build-env AS final +COPY --from=data /roq_basis /roq_basis +RUN mamba info +RUN python --version diff --git a/containers/dockerfile-template b/containers/dockerfile-template deleted file mode 100644 index f0ef592f0..000000000 --- a/containers/dockerfile-template +++ /dev/null @@ -1,25 +0,0 @@ -FROM containers.ligo.org/docker/base:conda -LABEL name="bilby CI testing" \ -maintainer="Gregory Ashton , Colm Talbot " - -COPY env-template.yml env.yml -RUN echo " - python={python_major_version}.{python_minor_version}" >> env.yml -ENV conda_env python{python_major_version}{python_minor_version} - -RUN mamba env create -f env.yml -n ${{conda_env}} -RUN echo "source activate ${{conda_env}}" > ~/.bashrc -ENV PATH /opt/conda/envs/${{conda_env}}/bin:$PATH -RUN /bin/bash -c "source activate ${{conda_env}}" -RUN mamba info -RUN python --version - -# Add the ROQ data to the image -RUN mkdir roq_basis \ - && cd roq_basis \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_addcal.hdf5 \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_multiband_addcal.hdf5 diff --git a/containers/env-template.yml b/containers/environment.yml similarity index 91% rename from containers/env-template.yml rename to containers/environment.yml index 42b3f3f8a..969155b75 100644 --- a/containers/env-template.yml +++ b/containers/environment.yml @@ -2,19 +2,17 @@ # The python version should be added before creating the env channels: - conda-forge - - defaults - pytorch dependencies: - pip - setuptools - setuptools_scm - matplotlib - - numpy<2.2 # dynesty<2.2 requires numpy<2.2 + - numpy<2.4 - scipy - pandas - astropy - flake8 - - anaconda - coverage - configargparse - future @@ -23,7 +21,7 @@ dependencies: - pytest-cov - pytest-requires - pytest-rerunfailures - - conda-forge::arviz + - arviz - parameterized - scikit-image - celerite @@ -37,7 +35,7 @@ dependencies: - bilby.cython - pyseobnr - ligo-gracedb - - gwpy + - gwpy>=4.0.0 - ligo.skymap - sphinx<9 # needed for sphinx_rtd_theme<3.1 - numpydoc diff --git a/containers/v3-dockerfile-test-suite-python310 b/containers/v3-dockerfile-test-suite-python310 deleted file mode 100644 index b7cbab298..000000000 --- a/containers/v3-dockerfile-test-suite-python310 +++ /dev/null @@ -1,27 +0,0 @@ -# This dockerfile is written automatically and should not be modified by hand. - -FROM containers.ligo.org/docker/base:conda -LABEL name="bilby CI testing" \ -maintainer="Gregory Ashton , Colm Talbot " - -COPY env-template.yml env.yml -RUN echo " - python=3.10" >> env.yml -ENV conda_env python310 - -RUN mamba env create -f env.yml -n ${conda_env} -RUN echo "source activate ${conda_env}" > ~/.bashrc -ENV PATH /opt/conda/envs/${conda_env}/bin:$PATH -RUN /bin/bash -c "source activate ${conda_env}" -RUN mamba info -RUN python --version - -# Add the ROQ data to the image -RUN mkdir roq_basis \ - && cd roq_basis \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_addcal.hdf5 \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_multiband_addcal.hdf5 diff --git a/containers/v3-dockerfile-test-suite-python311 b/containers/v3-dockerfile-test-suite-python311 deleted file mode 100644 index 85d27d405..000000000 --- a/containers/v3-dockerfile-test-suite-python311 +++ /dev/null @@ -1,27 +0,0 @@ -# This dockerfile is written automatically and should not be modified by hand. - -FROM containers.ligo.org/docker/base:conda -LABEL name="bilby CI testing" \ -maintainer="Gregory Ashton , Colm Talbot " - -COPY env-template.yml env.yml -RUN echo " - python=3.11" >> env.yml -ENV conda_env python311 - -RUN mamba env create -f env.yml -n ${conda_env} -RUN echo "source activate ${conda_env}" > ~/.bashrc -ENV PATH /opt/conda/envs/${conda_env}/bin:$PATH -RUN /bin/bash -c "source activate ${conda_env}" -RUN mamba info -RUN python --version - -# Add the ROQ data to the image -RUN mkdir roq_basis \ - && cd roq_basis \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_addcal.hdf5 \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_multiband_addcal.hdf5 diff --git a/containers/v3-dockerfile-test-suite-python312 b/containers/v3-dockerfile-test-suite-python312 deleted file mode 100644 index 60c0228d1..000000000 --- a/containers/v3-dockerfile-test-suite-python312 +++ /dev/null @@ -1,27 +0,0 @@ -# This dockerfile is written automatically and should not be modified by hand. - -FROM containers.ligo.org/docker/base:conda -LABEL name="bilby CI testing" \ -maintainer="Gregory Ashton , Colm Talbot " - -COPY env-template.yml env.yml -RUN echo " - python=3.12" >> env.yml -ENV conda_env python312 - -RUN mamba env create -f env.yml -n ${conda_env} -RUN echo "source activate ${conda_env}" > ~/.bashrc -ENV PATH /opt/conda/envs/${conda_env}/bin:$PATH -RUN /bin/bash -c "source activate ${conda_env}" -RUN mamba info -RUN python --version - -# Add the ROQ data to the image -RUN mkdir roq_basis \ - && cd roq_basis \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_linear.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/fnodes_quadratic.npy \ - && wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/params.dat \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_addcal.hdf5 \ - && wget https://git.ligo.org/soichiro.morisaki/roq_basis/raw/main/IMRPhenomD/16s_nospins/basis_multiband_addcal.hdf5 diff --git a/containers/write_dockerfiles.py b/containers/write_dockerfiles.py deleted file mode 100644 index 064c5d0f1..000000000 --- a/containers/write_dockerfiles.py +++ /dev/null @@ -1,23 +0,0 @@ -from datetime import date - -with open("dockerfile-template", "r") as ff: - template = ff.read() - -python_versions = [(3, 10), (3, 11), (3, 12)] -today = date.today().strftime("%Y%m%d") - -for python_major_version, python_minor_version in python_versions: - key = f"python{python_major_version}{python_minor_version}" - with open( - f"v3-dockerfile-test-suite-{key}", - "w" - ) as ff: - ff.write( - "# This dockerfile is written automatically and should not be " - "modified by hand.\n\n" - ) - ff.write(template.format( - date=today, - python_major_version=python_major_version, - python_minor_version=python_minor_version, - ))