Rewrite tiff reading #5710
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | ||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| tags: | ||
| - "**" | ||
| pull_request: {} | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.ref }} | ||
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | ||
| jobs: | ||
| changes: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| webknossos: ${{ steps.filter.outputs.webknossos || github.ref == 'refs/heads/master' }} | ||
| cluster_tools: ${{ steps.filter.outputs.cluster_tools || github.ref == 'refs/heads/master' }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: dorny/paths-filter@v2 | ||
| id: filter | ||
| with: | ||
| filters: | | ||
| webknossos: | ||
| - 'webknossos/**' | ||
| cluster_tools: | ||
| - 'cluster_tools/**' | ||
| cluster_tools_slurm: | ||
| needs: changes | ||
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 30 | ||
| strategy: | ||
| matrix: | ||
| python-version: ["3.13", "3.12", "3.11", "3.10"] | ||
| defaults: | ||
| run: | ||
| working-directory: cluster_tools | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: astral-sh/setup-uv@v6 | ||
| with: | ||
| enable-cache: true | ||
| cache-dependency-glob: "cluster_tools/uv.lock" | ||
| - run: uv python install ${{ matrix.python-version }} | ||
| - name: Start Docker Cluster | ||
| run: cd ./dockered-slurm && docker compose up -d | ||
| - name: Log Core Container | ||
| run: | | ||
| for name in "slurmctld" "c1" "c2"; do | ||
| docker logs "$name" | ||
| done | ||
| - name: Install UV dependencies | ||
| run: | | ||
| for name in "slurmctld" "c1" "c2"; do | ||
| docker exec -w /cluster_tools "$name" bash -c "uv sync --frozen" | ||
| done | ||
| - name: "Run Tests (test_all, test_slurm) without modified slurm.conf" | ||
| run: | | ||
| docker exec \ | ||
| -w /cluster_tools/tests \ | ||
| -e PYTEST_EXECUTORS=slurm \ | ||
| slurmctld bash -c "uv run --frozen python -m pytest -sv test_all.py test_slurm.py -m 'not requires_modified_slurm_config'" | ||
| - name: "Run Tests (test_deref_main)" | ||
| run: | | ||
| docker exec \ | ||
| -w /cluster_tools/tests \ | ||
| slurmctld bash -c "uv run --frozen python test_deref_main.py" | ||
| - name: Update Slurm Config | ||
| run: | | ||
| echo "MaxArraySize=2" >> ./dockered-slurm/slurm.conf | ||
| sed "s/JobAcctGatherFrequency=30/JobAcctGatherFrequency=1/g" ./dockered-slurm/slurm.conf > ./dockered-slurm/slurm.conf.tmp | ||
| mv ./dockered-slurm/slurm.conf.tmp ./dockered-slurm/slurm.conf | ||
| - name: Restart Slurm Cluster | ||
| run: cd ./dockered-slurm && docker compose restart slurmctld c1 c2 | ||
| - name: "Run Tests (test_all, test_slurm) with modified slurn.conf" | ||
| run: | | ||
| # Run tests requiring a modified slurm config | ||
| docker exec \ | ||
| -w /cluster_tools/tests \ | ||
| -e PYTEST_EXECUTORS=slurm \ | ||
| slurmctld bash -c "uv run --frozen python -m pytest -sv test_slurm.py -m 'requires_modified_slurm_config'" | ||
| cluster_tools_multiprocessing: | ||
| needs: changes | ||
| if: ${{ needs.changes.outputs.cluster_tools == 'true' }} | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 30 | ||
| strategy: | ||
| matrix: | ||
| python-version: ["3.13", "3.12", "3.11", "3.10"] | ||
| defaults: | ||
| run: | ||
| working-directory: cluster_tools | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@v6 | ||
| with: | ||
| enable-cache: true | ||
| cache-dependency-glob: "cluster_tools/uv.lock" | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| run: uv python install ${{ matrix.python-version }} | ||
| - name: Install dependencies (without docker) | ||
| run: uv sync --frozen | ||
| - name: Check typing | ||
| if: ${{ matrix.python-version == '3.11' }} | ||
| run: ./typecheck.sh | ||
| - name: Check formatting | ||
| if: ${{ matrix.python-version == '3.11' }} | ||
| run: ./format.sh check | ||
| - name: Lint code | ||
| if: ${{ matrix.python-version == '3.11' }} | ||
| run: ./lint.sh | ||
| - name: Run multiprocessing tests | ||
| run: | | ||
| cd tests | ||
| PYTEST_EXECUTORS=multiprocessing,sequential,multiprocessing_with_pickling,sequential_with_pickling \ | ||
| uv run --frozen python -m pytest -sv test_all.py test_multiprocessing.py | ||
| webknossos_linux: | ||
| needs: changes | ||
| if: | | ||
|
Check warning on line 129 in .github/workflows/ci.yml
|
||
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | ||
| ${{ needs.changes.outputs.webknossos == 'true' }} | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| matrix: | ||
| python-version: ["3.13", "3.12", "3.11", "3.10"] | ||
| group: [1, 2, 3] | ||
| fail-fast: false | ||
| defaults: | ||
| run: | ||
| working-directory: webknossos | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@v3 | ||
| with: | ||
| enable-cache: true | ||
| cache-dependency-glob: "webknossos/uv.lock" | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| run: uv python install ${{ matrix.python-version }} | ||
| - name: Check formatting | ||
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | ||
| run: ./format.sh check | ||
| - name: Lint code | ||
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | ||
| run: ./lint.sh | ||
| - name: Check typing | ||
| if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }} | ||
| run: ./typecheck.sh | ||
| - name: Patch Python standard library to assert that fork is not allowed | ||
| run: | | ||
| sed -i '/def _launch/a\ \ \ \ \ \ \ \ raise Exception("fork is not allowed.")' /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py | ||
| cat /home/runner/.local/share/uv/python/cpython-${{ matrix.python-version }}.*-linux-x86_64-gnu/lib/python${{ matrix.python-version }}/multiprocessing/popen_fork.py | ||
| - name: Python tests (with coverage) | ||
| if: ${{ matrix.python-version == '3.11' }} | ||
| timeout-minutes: 30 | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: uv run test.py --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration --cov=webknossos --cov-report= | ||
| - name: Python tests | ||
| if: ${{ matrix.python-version != '3.11' }} | ||
| timeout-minutes: 30 | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| run: uv run test.py --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | ||
| - name: Persist coverage reports | ||
| if: ${{ matrix.python-version == '3.11' }} | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: coverage-report-${{ matrix.group }} | ||
| path: "webknossos/.coverage" | ||
| include-hidden-files: true | ||
| if-no-files-found: error | ||
| retention-days: 1 # Automatically deletes after 1 day | ||
| - name: Check if git is dirty | ||
| run: ../_tooling/check_clean.sh | ||
| webknossos_windows: | ||
| needs: changes | ||
| if: | | ||
|
Check warning on line 200 in .github/workflows/ci.yml
|
||
| ${{ needs.changes.outputs.cluster_tools == 'true' }} || | ||
| ${{ needs.changes.outputs.webknossos == 'true' }} | ||
| runs-on: windows-latest | ||
| strategy: | ||
| matrix: | ||
| python-version: ["3.12"] | ||
| group: [1, 2, 3] | ||
| fail-fast: false | ||
| defaults: | ||
| run: | ||
| working-directory: webknossos | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@v3 | ||
| with: | ||
| enable-cache: true | ||
| cache-dependency-glob: "webknossos/uv.lock" | ||
| - name: Install minio | ||
| run: Invoke-WebRequest -Uri "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" -OutFile "minio.exe" | ||
| - name: Set up Python ${{ matrix.python-version }} | ||
| run: uv python install ${{ matrix.python-version }} | ||
| - name: Python tests | ||
| timeout-minutes: 30 | ||
| env: | ||
| PYTHON_VERSION: ${{ matrix.python-version }} | ||
| MULTIPROCESSING_DEFAULT_START_METHOD: spawn | ||
| run: uv run test.py --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration | ||
| coverage_report: | ||
| needs: [webknossos_linux] | ||
| if: success() | ||
| runs-on: ubuntu-latest | ||
| defaults: | ||
| run: | ||
| working-directory: webknossos | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Install coverage.py | ||
| run: pip install coverage | ||
| - name: Download all coverage reports | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| path: ~/coverage-files | ||
| - name: Generate coverage report | ||
| env: | ||
| COVERAGE_FILE: "~/coverage-files/result.coverage" | ||
| run: | | ||
| coverage combine ~/coverage-files/**/.coverage | ||
| coverage xml -i -o ~/coverage-files/result.xml | ||
| - name: Report coverage | ||
| uses: orgoro/coverage@v3.2 | ||
| with: | ||
| coverageFile: /home/runner/coverage-files/result.xml | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| thresholdAll: 0.8 | ||
| - name: Cleanup temporary files | ||
| run: rm -rf ~/coverage-files | ||
| docs: | ||
| needs: | ||
| - cluster_tools_slurm | ||
| - cluster_tools_multiprocessing | ||
| - webknossos_linux | ||
| - webknossos_windows | ||
| runs-on: ubuntu-latest | ||
| if: | | ||
| always() && | ||
| !contains(needs.*.result, 'failure') && | ||
| !contains(needs.*.result, 'cancelled') && | ||
| !github.event.pull_request.head.repo.fork | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| repository: scalableminds/webknossos | ||
| path: docs/wk-repo | ||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@v3 | ||
| - name: Build Docs | ||
| run: | | ||
| cd docs | ||
| ./generate.sh --persist | ||
| - name: Push docs (for master) | ||
| if: github.ref == 'refs/heads/master' | ||
| env: | ||
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
| run: | | ||
| CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME} | ||
| NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_} | ||
| uvx --from awscli aws s3 cp --recursive --endpoint-url https://hel1.your-objectstorage.com docs/out s3://webknossos-www/docs/${NORMALIZED_CI_BRANCH} | ||
| - name: Push docs (for tag) | ||
| if: github.ref_type == 'tag' | ||
| env: | ||
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
| run: | | ||
| CI_TAG=$(git describe --tags) | ||
| uvx --from awscli aws s3 cp --recursive --endpoint-url https://hel1.your-objectstorage.com docs/out s3://webknossos-www/docs/${CI_TAG} | ||
| - name: Check links (on master) | ||
| if: github.ref == 'refs/heads/master' | ||
| env: # Or as an environment variable | ||
| SLACK_HOOK: ${{ secrets.LINK_CHECKER_SLACK_HOOK }} | ||
| run: | | ||
| cd docs | ||
| uv run --frozen linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \ | ||
| curl -X POST --data-urlencode "payload={\"text\": \":warning: Broken Links on doc.webknossos.org :warning:\n"'```'"\n$(cat link_status)\n"'```"}' \ | ||
| "$SLACK_HOOK" | ||
| pypi_and_gh_release: | ||
| needs: | ||
| - cluster_tools_slurm | ||
| - cluster_tools_multiprocessing | ||
| - webknossos_linux | ||
| - webknossos_windows | ||
| if: | | ||
| always() && | ||
| !contains(needs.*.result, 'failure') && | ||
| !contains(needs.*.result, 'cancelled') && | ||
| github.ref_type == 'tag' && | ||
| !github.event.pull_request.head.repo.fork | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Install uv | ||
| uses: astral-sh/setup-uv@v3 | ||
| - name: Publish python packages | ||
| env: | ||
| UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }} | ||
| run: _tooling/publish.sh | ||
| - name: Prepare github release | ||
| run: | | ||
| VERSION="$(uvx dunamai from git)" | ||
| _tooling/changelog_for_version.sh $VERSION > Changelog.md | ||
| - name: Publish github release | ||
| id: create_release | ||
| uses: actions/create-release@v1 | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| with: | ||
| tag_name: ${{ github.ref }} | ||
| release_name: ${{ github.ref }} | ||
| body_path: Changelog.md | ||
| draft: false | ||
| prerelease: false | ||
| complete: | ||
| needs: | ||
| - cluster_tools_multiprocessing | ||
| - cluster_tools_slurm | ||
| - webknossos_linux | ||
| - webknossos_windows | ||
| - docs | ||
| - pypi_and_gh_release | ||
| if: always() | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Check failure | ||
| if: | | ||
| contains(needs.*.result, 'failure') || | ||
| contains(needs.*.result, 'cancelled') | ||
| run: exit 1 | ||
| - name: Success | ||
| run: echo Success! | ||
