diff --git a/.github/workflows/skip-tests.yml b/.github/workflows/skip-tests.yml deleted file mode 100644 index 2a3107c5d..000000000 --- a/.github/workflows/skip-tests.yml +++ /dev/null @@ -1,101 +0,0 @@ ---- - -# We check that using all --skip options does not error. - -name: Skip tests - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - # * is a special character in YAML so you have to quote this string - # Run at 1:00 every day - - cron: 0 1 * * * - workflow_dispatch: {} - -jobs: - build: - - strategy: - matrix: - python-version: ['3.13'] - platform: [ubuntu-latest] - - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/checkout@v5 - with: - # See https://github.com/codecov/codecov-action/issues/190. - fetch-depth: 2 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - cache-dependency-glob: '**/pyproject.toml' - - - name: Set secrets file - run: | - cp ./vuforia_secrets.env.example ./vuforia_secrets.env - - - name: Run tests - run: | - uv run --extra=dev pytest \ - --skip-docker_build_tests \ - --skip-docker_in_memory \ - --skip-mock \ - --skip-real \ - --capture=no \ - -vvv \ - --exitfirst \ - --cov=src/ \ - --cov=tests/ \ - --cov-report=xml \ - . - env: - UV_PYTHON: ${{ matrix.python-version }} - - - name: Show coverage file - run: | - # Sometimes we have been sure that we have 100% coverage, but codecov - # says otherwise. - # - # We show the coverage file here to help with debugging. - # https://github.com/VWS-Python/vws-python-mock/issues/708 - cat ./coverage.xml - - # We run this job on every PR, on every merge to main, and nightly. - # This causes us to hit an issue with Codecov. - # - # We see "Too many uploads to this commit.". - # See https://community.codecov.io/t/too-many-uploads-to-this-commit/2574. - # - # To work around this, we do not upload coverage data on scheduled runs. - # We print the event name here to help with debugging. - - name: Show event name - run: | - echo ${{ github.event_name }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - fail_ci_if_error: true - # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - # which tells us to use the token to avoid errors. - token: ${{ secrets.CODECOV_TOKEN }} - if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} - - completion-skip-tests: - needs: build - runs-on: ubuntu-latest - if: always() # Run even if one matrix job fails - steps: - - name: Check matrix job status - run: |- - if ! ${{ needs.build.result == 'success' }}; then - echo "One or more matrix jobs failed" - exit 1 - fi diff --git a/.github/workflows/ci.yml b/.github/workflows/test.yml similarity index 61% rename from .github/workflows/ci.yml rename to .github/workflows/test.yml index 676ccd6e4..1042c73d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,5 @@ --- - -name: CI +name: Test on: push: @@ -14,15 +13,13 @@ on: workflow_dispatch: {} # We share Vuforia credentials and therefore Vuforia databases across -# workflows. -# We therefore want to run only one workflow at a time. +# workflows. We therefore want to run only one workflow at a time. concurrency: vuforia_credentials jobs: - build: - + # CI tests with matrix + ci-tests: runs-on: ubuntu-latest - strategy: fail-fast: false matrix: @@ -120,9 +117,6 @@ jobs: steps: - uses: actions/checkout@v5 - with: - # See https://github.com/codecov/codecov-action/issues/190. - fetch-depth: 2 - name: Install uv uses: astral-sh/setup-uv@v6 @@ -158,54 +152,189 @@ jobs: - name: Run tests run: | - uv run --extra=dev pytest \ - -s \ - -vvv \ - --showlocals \ - --exitfirst \ - --cov=src/ \ - --cov=tests/ \ - --cov-report=xml \ - ${{ matrix.ci_pattern }} + uv run --extra=dev \ + coverage run \ + --parallel-mode \ + --source=src/ \ + --source=tests/ \ + -m pytest \ + -s \ + -vvv \ + --showlocals \ + --exitfirst \ + ${{ matrix.ci_pattern }} env: UV_PYTHON: ${{ matrix.python-version }} - - name: Show coverage file + - name: Sanitize pattern for artifact name + id: sanitize + run: | + SANITIZED_PATTERN=$(echo "${{ matrix.ci_pattern }}" | sed 's/::/-/g' | sed 's/:/-/g' | sed 's|/|-|g') + echo "name=coverage-data-ci-${{ matrix.python-version }}-${SANITIZED_PATTERN}" >> "$GITHUB_OUTPUT" + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.sanitize.outputs.name }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: error + + # Skip tests + skip-tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.13'] + platform: [ubuntu-latest] + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: '**/pyproject.toml' + + - name: Set secrets file + run: | + cp ./vuforia_secrets.env.example ./vuforia_secrets.env + + - name: Run tests run: | - # Sometimes we have been sure that we have 100% coverage, but codecov - # says otherwise. + uv run --extra=dev \ + coverage run \ + --parallel-mode \ + --source=src/ \ + --source=tests/ \ + -m pytest \ + --skip-docker_build_tests \ + --skip-docker_in_memory \ + --skip-mock \ + --skip-real \ + --capture=no \ + -vvv \ + --exitfirst \ + . + env: + UV_PYTHON: ${{ matrix.python-version }} + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: coverage-data-skip-tests-${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: error + + # Windows tests + windows-tests: + runs-on: windows-latest + strategy: + matrix: + python-version: ['3.13'] + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: '**/pyproject.toml' + + - name: Set secrets file + run: | + cp ./vuforia_secrets.env.example ./vuforia_secrets.env + + - name: Run tests + shell: bash + run: | + # We use pytest-xdist to make this run much faster. + # The downside is that we cannot use -s / --capture=no. + # + # We use coverage to collect coverage data but we currently + # do not upload / use it because combining Windows and Linux + # coverage is challenging. # - # We show the coverage file here to help with debugging. - # https://github.com/VWS-Python/vws-python-mock/issues/708 - cat ./coverage.xml - - # We run this job on every PR, on every merge to main, and nightly. - # This causes us to hit an issue with Codecov. - # - # We see "Too many uploads to this commit.". - # See https://community.codecov.io/t/too-many-uploads-to-this-commit/2574. - # - # To work around this, we do not upload coverage data on scheduled runs. - # We print the event name here to help with debugging. - - name: Show event name + # We therefore have a few ``# pragma: no cover`` statements. + uv run --extra=dev \ + coverage run \ + --parallel-mode \ + --source=src/ \ + --source=tests/ \ + -m pytest \ + --skip-real \ + -vvv \ + --exitfirst \ + -n auto \ + . + env: + UV_PYTHON: ${{ matrix.python-version }} + + coverage: + name: Combine & check coverage + needs: [ci-tests, skip-tests, windows-tests] + if: always() + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: '**/pyproject.toml' + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-* + merge-multiple: true + + - name: Require 100% Coverage + id: coverage run: | - echo ${{ github.event_name }} + uv tool install 'coverage[toml]' - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 + coverage combine + coverage html --skip-covered --skip-empty + + # Report and write to summary. + coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + + # Report again and fail if under 100%. + coverage report --fail-under=100 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 with: - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + name: html-report + path: htmlcov + if: ${{ failure() }} - completion-ci: - needs: build + # Final completion check + completion: + needs: [ci-tests, skip-tests, windows-tests, coverage] runs-on: ubuntu-latest - if: always() # Run even if one matrix job fails + if: always() steps: - - name: Check matrix job status + - name: Check all jobs status run: |- - if ! ${{ needs.build.result == 'success' }}; then - echo "One or more matrix jobs failed" + if ! ${{ needs.ci-tests.result == 'success' }}; then + echo "CI tests failed" + exit 1 + fi + if ! ${{ needs.skip-tests.result == 'success' }}; then + echo "Skip tests failed" + exit 1 + fi + if ! ${{ needs.windows-tests.result == 'success' }}; then + echo "Windows tests failed" + exit 1 + fi + if ! ${{ needs.coverage.result == 'success' }}; then + echo "Coverage check failed" exit 1 fi diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml deleted file mode 100644 index 3dd3ff0a2..000000000 --- a/.github/workflows/windows-ci.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- - -name: Windows CI - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - # * is a special character in YAML so you have to quote this string - # Run at 1:00 every day - - cron: 0 1 * * * - workflow_dispatch: {} - -jobs: - build: - - strategy: - matrix: - python-version: ['3.13'] - platform: [windows-latest] - - runs-on: ${{ matrix.platform }} - - steps: - - uses: actions/checkout@v5 - with: - # See https://github.com/codecov/codecov-action/issues/190. - fetch-depth: 2 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - cache-dependency-glob: '**/pyproject.toml' - - - name: Set secrets file - run: | - cp ./vuforia_secrets.env.example ./vuforia_secrets.env - - - name: Run tests - run: | - # We use pytest-xdist to make this run much faster. - # The downside is that we cannot use -s / --capture=no. - uv run --extra=dev pytest --skip-real -vvv --exitfirst -n auto --cov=src/ --cov=tests/ --cov-report=xml . - env: - UV_PYTHON: ${{ matrix.python-version }} - - - name: Show coverage file - run: | - # Sometimes we have been sure that we have 100% coverage, but codecov - # says otherwise. - # - # We show the coverage file here to help with debugging. - # https://github.com/VWS-Python/vws-python-mock/issues/708 - cat ./coverage.xml - - # We run this job on every PR, on every merge to main, and nightly. - # This causes us to hit an issue with Codecov. - # - # We see "Too many uploads to this commit.". - # See https://community.codecov.io/t/too-many-uploads-to-this-commit/2574. - # - # To work around this, we do not upload coverage data on scheduled runs. - # We print the event name here to help with debugging. - - name: Show event name - run: | - echo ${{ github.event_name }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - fail_ci_if_error: true - # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - # which tells us to use the token to avoid errors. - token: ${{ secrets.CODECOV_TOKEN }} - if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} - - completion-windows-ci: - needs: build - runs-on: ubuntu-latest - if: always() # Run even if one matrix job fails - steps: - - name: Check matrix job status - run: |- - if ! ${{ needs.build.result == 'success' }}; then - echo "One or more matrix jobs failed" - exit 1 - fi diff --git a/README.rst b/README.rst index c2a566463..98004f17c 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -|Build Status| |codecov| |PyPI| +|Build Status| |PyPI| VWS Mock ======== @@ -56,8 +56,6 @@ This includes details on how to use the mock, options, and details of the differ .. |Build Status| image:: https://github.com/VWS-Python/vws-python-mock/actions/workflows/ci.yml/badge.svg?branch=main :target: https://github.com/VWS-Python/vws-python-mock/actions -.. |codecov| image:: https://codecov.io/gh/VWS-Python/vws-python-mock/branch/main/graph/badge.svg - :target: https://codecov.io/gh/VWS-Python/vws-python-mock .. |PyPI| image:: https://badge.fury.io/py/VWS-Python-Mock.svg :target: https://badge.fury.io/py/VWS-Python-Mock .. |minimum-python-version| replace:: 3.13 diff --git a/ci/test_custom_linters.py b/ci/test_custom_linters.py index 8de79cbb8..f740f39a0 100644 --- a/ci/test_custom_linters.py +++ b/ci/test_custom_linters.py @@ -18,9 +18,9 @@ def _ci_patterns(*, repository_root: Path) -> set[str]: """ Return the CI patterns given in the CI configuration file. """ - ci_file = repository_root / ".github" / "workflows" / "ci.yml" + ci_file = repository_root / ".github" / "workflows" / "test.yml" github_workflow_config = yaml.safe_load(stream=ci_file.read_text()) - matrix = github_workflow_config["jobs"]["build"]["strategy"]["matrix"] + matrix = github_workflow_config["jobs"]["ci-tests"]["strategy"]["matrix"] ci_pattern_list = matrix["ci_pattern"] ci_patterns = set(ci_pattern_list) assert len(ci_pattern_list) == len(ci_patterns) diff --git a/codecov.yaml b/codecov.yaml deleted file mode 100644 index 5c35baac9..000000000 --- a/codecov.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -coverage: - status: - patch: - default: - # Require 100% test coverage. - target: 100% diff --git a/pyproject.toml b/pyproject.toml index c2da135d8..051535a80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ optional-dependencies.dev = [ "actionlint-py==1.7.7.23", "check-manifest==0.50", "check-wheel-contents==0.6.3", + "coverage==7.10.7", "deptry==0.23.1", "dirty-equals==0.10.0", "doc8==1.1.1", @@ -72,7 +73,6 @@ optional-dependencies.dev = [ "pyright==1.1.406", "pyroma==5.0", "pytest==8.4.2", - "pytest-cov==7.0.0", "pytest-retry==1.7.0", "pytest-xdist==3.8.0", "python-dotenv==1.1.1", @@ -313,7 +313,6 @@ ignore = [ "Makefile", "ci", "ci/**", - "codecov.yaml", "docs", "docs/**", ".git_archival.txt", diff --git a/tests/mock_vws/test_docker.py b/tests/mock_vws/test_docker.py index 56c5fc6ce..d3d18c16a 100644 --- a/tests/mock_vws/test_docker.py +++ b/tests/mock_vws/test_docker.py @@ -65,7 +65,9 @@ def fixture_custom_bridge_network() -> Iterator[Network]: name = "test-vws-bridge-" + uuid.uuid4().hex try: network = client.networks.create(name=name, driver="bridge") - except NotFound: + # We skip coverage here because combining Windows and Linux coverage + # is challenging. + except NotFound: # pragma: no cover # On Windows the "bridge" network driver is not available and we use # the "nat" driver instead. network = client.networks.create(name=name, driver="nat") @@ -116,15 +118,15 @@ def test_build_and_run( target="target-manager", rm=True, ) - except BuildError as exc: + # We skip coverage here because combining Windows and Linux coverage + # is challenging. + except BuildError as exc: # pragma: no cover full_log = "\n".join( [item["stream"] for item in exc.build_log if "stream" in item], ) # If this assertion fails, it may be useful to look at the other # properties of ``exc``. - if ( - "no matching manifest for windows/amd64" not in exc.msg - ): # pragma: no cover + if "no matching manifest for windows/amd64" not in exc.msg: raise AssertionError(full_log) from exc pytest.skip( reason="We do not currently support using Windows containers."