Skip to content

Commit 5610788

Browse files
ColmTalbotColmTalbot
authored
Fix and test containers (#1067)
* BLD: add basic testing to containers * BLD: try cleaning source from docker build * BLD: add setuptools_scm catch for dockerfile * Pin arviz version in test environment * Fix setuptools dubious owner error in unit test job
1 parent cde65c5 commit 5610788

6 files changed

Lines changed: 127 additions & 37 deletions

File tree

‎.github/workflows/basic-install.yml‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -37,23 +37,9 @@ jobs:
3737
- name: List environment
3838
run: python -m pip list installed
3939
- name: Test imports
40+
run: bash test/ci_test_imports.sh
41+
- name: Test entry points
4042
run: |
41-
python -c "import bilby"
42-
python -c "import bilby.bilby_mcmc"
43-
python -c "import bilby.core"
44-
python -c "import bilby.core.prior"
45-
python -c "import bilby.core.sampler"
46-
python -c "import bilby.core.utils"
47-
python -c "import bilby.gw"
48-
python -c "import bilby.gw.detector"
49-
python -c "import bilby.gw.eos"
50-
python -c "import bilby.gw.likelihood"
51-
python -c "import bilby.gw.sampler"
52-
python -c "import bilby.hyper"
53-
python -c "import cli_bilby"
54-
python test/import_test.py
55-
# - if: ${{ matrix.os != "windows-latest" }}
56-
# run: |
57-
# for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do
58-
# ${script} --help;
59-
# done
43+
for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do
44+
${script} --help;
45+
done

‎.github/workflows/build-containers.yml‎

Lines changed: 96 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,22 @@ name: Build images
22

33
on:
44
schedule:
5-
- cron: "15 0 * * *"
5+
- cron: "15 0 * * *"
66
workflow_dispatch:
77

88
jobs:
99
build-container:
1010
if: ${{ github.repository == 'bilby-dev/bilby' }}
1111
permissions:
12-
attestations: write
1312
contents: read
14-
id-token: write
15-
packages: write
1613
runs-on: ubuntu-latest
1714
strategy:
1815
fail-fast: false
1916
matrix:
2017
python-version: ["11", "12", "13"]
2118
env:
2219
LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }}
20+
IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar
2321
steps:
2422
- uses: actions/checkout@v4
2523

@@ -30,33 +28,114 @@ jobs:
3028
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
3129
df . -h
3230
33-
- name: Login to the Container registry
34-
uses: docker/login-action@v3
35-
with:
36-
registry: ghcr.io
37-
username: ${{ github.actor }}
38-
password: ${{ secrets.GITHUB_TOKEN }}
39-
4031
- name: Set up QEMU
4132
uses: docker/setup-qemu-action@v3
4233

4334
- name: Set up Docker Buildx
4435
uses: docker/setup-buildx-action@v3
4536

46-
- name: Build and push Docker image
47-
id: push
37+
- name: Build Docker image archive
4838
uses: docker/build-push-action@v6
4939
with:
5040
context: .
5141
build-args: |
5242
python_minor_version=${{ matrix.python-version }}
5343
ENV_FILE=containers/environment.yml
54-
push: true
44+
push: false
5545
file: containers/Dockerfile
5646
tags: ${{ env.LABEL }}:latest
57-
cache-from: type=registry,ref=${{ env.LABEL }}:buildcache
58-
cache-to: type=registry,ref=${{ env.LABEL }}:buildcache,mode=max
59-
47+
outputs: type=docker,dest=/tmp/${{ env.IMAGE_ARCHIVE }}
48+
cache-from: type=gha,scope=bilby-python3${{ matrix.python-version }}
49+
cache-to: type=gha,scope=bilby-python3${{ matrix.python-version }},mode=max
50+
51+
- name: Upload Docker image archive
52+
uses: actions/upload-artifact@v4
53+
with:
54+
name: bilby-python3${{ matrix.python-version }}-image
55+
path: /tmp/${{ env.IMAGE_ARCHIVE }}
56+
if-no-files-found: error
57+
58+
test-container:
59+
if: ${{ github.repository == 'bilby-dev/bilby' }}
60+
needs: build-container
61+
permissions:
62+
contents: read
63+
runs-on: ubuntu-latest
64+
strategy:
65+
fail-fast: false
66+
matrix:
67+
python-version: ["11", "12", "13"]
68+
env:
69+
LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }}
70+
IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar
71+
steps:
72+
- uses: actions/checkout@v4
73+
74+
- name: Download Docker image archive
75+
uses: actions/download-artifact@v4
76+
with:
77+
name: bilby-python3${{ matrix.python-version }}-image
78+
path: /tmp
79+
80+
- name: Load Docker image
81+
run: docker load --input /tmp/${{ env.IMAGE_ARCHIVE }}
82+
83+
- name: Smoke test and import checks
84+
run: |
85+
docker run --rm \
86+
-v "$PWD:/workspaces/bilby" \
87+
-w /workspaces/bilby \
88+
${{ env.LABEL }}:latest \
89+
bash -lc '
90+
set -e
91+
python -m pip install -e .
92+
bilby_result --help
93+
bash test/ci_test_imports.sh
94+
for script in $(pip show -f bilby | grep "bin\/" | xargs -I {} basename {}); do
95+
${script} --help;
96+
done
97+
'
98+
99+
push-container:
100+
if: ${{ github.repository == 'bilby-dev/bilby' }}
101+
needs: test-container
102+
permissions:
103+
attestations: write
104+
contents: read
105+
id-token: write
106+
packages: write
107+
runs-on: ubuntu-latest
108+
strategy:
109+
fail-fast: false
110+
matrix:
111+
python-version: ["11", "12", "13"]
112+
env:
113+
LABEL: ghcr.io/${{ github.repository }}-python3${{ matrix.python-version }}
114+
IMAGE_ARCHIVE: bilby-python3${{ matrix.python-version }}.tar
115+
steps:
116+
- name: Download Docker image archive
117+
uses: actions/download-artifact@v4
118+
with:
119+
name: bilby-python3${{ matrix.python-version }}-image
120+
path: /tmp
121+
122+
- name: Load Docker image
123+
run: docker load --input /tmp/${{ env.IMAGE_ARCHIVE }}
124+
125+
- name: Login to the Container registry
126+
uses: docker/login-action@v3
127+
with:
128+
registry: ghcr.io
129+
username: ${{ github.actor }}
130+
password: ${{ secrets.GITHUB_TOKEN }}
131+
132+
- name: Push Docker image
133+
id: push
134+
run: |
135+
docker push ${{ env.LABEL }}:latest
136+
digest=$(docker image inspect --format='{{index .RepoDigests 0}}' ${{ env.LABEL }}:latest | sed 's/.*@//')
137+
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
138+
60139
- name: Generate artifact attestation
61140
uses: actions/attest-build-provenance@v3
62141
with:

‎.github/workflows/unit-tests.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ concurrency:
2020
env:
2121
CONDA_PATH: /opt/conda/
2222
BILBY_ALLOW_PARAMETERS_AS_STATE: FALSE
23+
SETUPTOOLS_SCM_IGNORE_DUBIOUS_OWNER: 1
2324

2425
jobs:
2526
build:

‎containers/Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ RUN wget https://git.ligo.org/lscsoft/ROQ_data/raw/master/IMRPhenomPv2/4s/B_lin
2828
#############################
2929

3030
FROM os-deps AS build-env
31+
WORKDIR /bldenv
3132
ARG ENV_FILE=environment.yml
3233
COPY ${ENV_FILE} env.yml
3334
ARG python_major_version=3
@@ -40,6 +41,7 @@ RUN mamba env create -f env.yml -n ${conda_env}
4041
RUN echo "source activate ${conda_env}" > ~/.bashrc
4142
ENV PATH=/opt/conda/envs/${conda_env}/bin:$PATH
4243
RUN /bin/bash -c "source activate ${conda_env}"
44+
RUN cd / && rm -rf /bldenv
4345
RUN mamba clean --all
4446

4547
#############################
@@ -48,5 +50,7 @@ RUN mamba clean --all
4850

4951
FROM build-env AS final
5052
COPY --from=data /roq_basis /roq_basis
53+
# otherwise CI jobs don't like installing bilby
54+
ENV SETUPTOOLS_SCM_IGNORE_DUBIOUS_OWNER=1
5155
RUN mamba info
5256
RUN python --version

‎containers/environment.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ dependencies:
2121
- pytest-cov
2222
- pytest-requires
2323
- pytest-rerunfailures
24-
- arviz
24+
- arviz<1 # arviz removed InferenceData in version 1
2525
- parameterized
2626
- scikit-image
2727
- celerite

‎test/ci_test_imports.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
# test that the various imports within the package works and can be reused across workflows
4+
5+
set -euo pipefail
6+
7+
python -c "import bilby"
8+
python -c "import bilby.bilby_mcmc"
9+
python -c "import bilby.core"
10+
python -c "import bilby.core.prior"
11+
python -c "import bilby.core.sampler"
12+
python -c "import bilby.core.utils"
13+
python -c "import bilby.gw"
14+
python -c "import bilby.gw.detector"
15+
python -c "import bilby.gw.eos"
16+
python -c "import bilby.gw.likelihood"
17+
python -c "import bilby.gw.sampler"
18+
python -c "import bilby.hyper"
19+
python -c "import cli_bilby"
20+
python test/import_test.py

0 commit comments

Comments
 (0)