feat: smoke tests

.github/workflows/integration-tests-with-creds.yaml

@@ -32,6 +32,11 @@ jobs:
           envkey_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           envkey_QDRANT_URL: ${{ secrets.QDRANT_URL }}
           envkey_QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}
+          envkey_EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
+          envkey_E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
+          envkey_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          envkey_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          envkey_AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
 
       - name: Load Test Image
         uses: docker/bake-action@v5

.github/workflows/smoke-tests.yaml

@@ -0,0 +1,45 @@
+name: Smoke Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  smoke-tests:
+    name: Smoke Tests
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      (github.event.label.name == 'run-smoke-tests' &&
+       github.event.pull_request.head.repo.full_name == github.repository)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create .env file
+        uses: SpicyPizza/create-envfile@v2.0
+        with:
+          envkey_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          envkey_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          envkey_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          envkey_EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
+          envkey_E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
+          envkey_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          envkey_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          envkey_AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+
+      - name: Load Test Image
+        uses: docker/bake-action@v5
+        with:
+          load: true
+          set: |
+            *.cache-from=type=gha
+            *.cache-to=type=gha,mode=max
+          targets: dynamiq-app-test
+
+      - name: Run smoke tests
+        run: docker compose up dynamiq-app-test-smoke --exit-code-from dynamiq-app-test-smoke

Makefile

@@ -29,20 +29,26 @@ test-integration:
 	pytest tests/integration
 
 test-integration-with-creds:
-	pytest tests/integration_with_creds
+	pytest tests/integration_with_creds -m "not smoke"
+
+# Smoke tests: slow, paid end-to-end production scenarios. Excluded from every default target and
+# run ONLY here (gated behind the `run-smoke-tests` PR label in CI). -n auto parallelizes the
+# matrix; if E2B quota / provider 429s appear, dial back to `-n 4` or group same-provider cases.
+test-smoke:
+	pytest tests -m smoke -n auto --dist worksteal
 
 test-exclude-integration-with-creds:
-	pytest tests --ignore=tests/integration_with_creds
+	pytest tests --ignore=tests/integration_with_creds -m "not smoke"
 
 test-unit:
 	pytest tests/unit
 
 test:
-	pytest tests
+	pytest tests -m "not smoke"
 
 test-cov:
 	mkdir -p ./reports
-	coverage run -m pytest --junitxml=./reports/test-results.xml tests
+	coverage run -m pytest --junitxml=./reports/test-results.xml -m "not smoke" tests
 	coverage report --skip-empty --skip-covered
 	coverage html -d ./reports/htmlcov --omit="*/test_*,*/tests.py"
 	coverage xml -o ./reports/coverage.xml --omit="*/test_*,*/tests.py"

docker-compose.yaml

@@ -48,3 +48,13 @@ services:
       - .env
     volumes:
       - ./:/app
+
+  dynamiq-app-test-smoke:
+    image: dynamiq-app:${IMAGE_TAG:-local}
+    build:
+      target: develop
+    entrypoint: ["make", "test-smoke"]
+    env_file:
+      - .env
+    volumes:
+      - ./:/app

pyproject.toml

@@ -107,6 +107,8 @@ mkdocs-autorefs = "<1.4.0"
 griffe = "<2.0.0"
 requests-mock = "~1.12.1"
 pytest-rerunfailures = "^16.1"
+pytest-xdist = "^3.8.0"
+pytest-timeout = "^2.4.0"
 
 [tool.poetry.group.examples]
 optional = true
@@ -140,5 +142,12 @@ exclude_dirs = ["tests"]
 [tool.isort]
 line_length = 120
 
+[tool.pytest.ini_options]
+markers = [
+    "smoke: slow, paid end-to-end production-scenario tests; run only via the `run-smoke-tests` label / `make test-smoke`",
+    "integration: tests that exercise real integrations (may require credentials)",
+    "unit: fast, isolated unit tests",
+]
+
 [tool.poetry.scripts]
 dynamiq = "dynamiq.cli:main"