Add code coverage reporting to unit test workflow

.github/workflows/unit_tests.yml

@@ -51,10 +51,18 @@ jobs:
         with:
           python-version-file: ".python-version"
 
-      - name: Install dependencies and check code
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run tests with coverage
         run: |
           cd aieng-eval-agents
-          uv venv .venv
-          source .venv/bin/activate
-          uv sync --all-extras --dev
-          uv run pytest -m "not integration_test" tests
+          uv run pytest -m "not integration_test" --cov aieng/agent_evals --cov-report=xml tests
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: VectorInstitute/eval-agents
+          fail_ci_if_error: false
+          verbose: true

aieng-eval-agents/pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
 [dependency-groups]
 dev = [
     "pytest>=8.3.5",
+    "pytest-cov>=6.0.0",
 ]
 
 [build-system]

codecov.yml

@@ -0,0 +1,19 @@
+codecov:
+  require_ci_to_pass: true
+  notify:
+    after_n_builds: 2
+    wait_for_ci: yes
+comment:
+  behavior: default
+  layout: reach,diff,flags,tree,reach
+  show_carryforward_flags: false
+  require_changes: true
+coverage:
+  status:
+    changes: true
+    default_rules:
+      flag_coverage_not_uploaded_behavior: include
+    patch: true
+    project: true
+github_checks:
+  annotations: true