From ed727216d493638573e608f7bfcc9f8d265161ef Mon Sep 17 00:00:00 2001 From: Christian Schwartz Date: Fri, 20 Mar 2026 17:04:30 +0100 Subject: [PATCH 1/3] refactor: Migrate processes/services to devenv2 --- .gitignore | 5 +- mock-services/devenv.nix | 56 ++--- verdict-backend/app/database_config.py | 14 +- verdict-backend/devenv.nix | 199 ++++++------------ verdict-backend/justfile | 43 ---- verdict-backend/tests/e2e/conftest.py | 3 +- .../tests/routes/test_ingestion.py | 17 +- 7 files changed, 112 insertions(+), 225 deletions(-) diff --git a/.gitignore b/.gitignore index e94ab2f..0b974f2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,10 @@ devenv.local.nix .envrc # generated by devenv -.env.sample +.env.development +.env.test +# local overrides (never commit) +.env.*.local # AI tooling .claude diff --git a/mock-services/devenv.nix b/mock-services/devenv.nix index b1083ed..8d6e4e6 100644 --- a/mock-services/devenv.nix +++ b/mock-services/devenv.nix @@ -14,52 +14,36 @@ in }; # Points to the mock-services directory at runtime. - # Default works when mock-services IS the devenv project. - # Importers (e.g. verdict-backend) override via mkForce. - env.MOCK_SERVICES_DIR = lib.mkDefault config.devenv.root; + # Uses the git root so this works both standalone and when imported. + env.MOCK_SERVICES_DIR = lib.mkDefault "${config.git.root}/mock-services"; processes.asset-inventory-mock = { - exec = "cd $MOCK_SERVICES_DIR && uv run uvicorn asset_inventory.app:app --host 0.0.0.0 --port 4010"; - process-compose = { - readiness_probe = { - http_get = { - host = "localhost"; - port = 4010; - path = "/assets"; - }; - initial_delay_seconds = 2; - period_seconds = 2; - }; + exec = "cd $MOCK_SERVICES_DIR && uv run uvicorn asset_inventory.app:app --host 0.0.0.0 --port ${toString config.processes.asset-inventory-mock.ports.http.value}"; + ports.http.allocate = 4010; + ready.http.get = { + host = "localhost"; + port = config.processes.asset-inventory-mock.ports.http.value; + path = "/assets"; }; }; processes.cmdb-mock = { - exec = "cd $MOCK_SERVICES_DIR && uv run uvicorn cmdb.app:app --host 0.0.0.0 --port 4011"; - process-compose = { - readiness_probe = { - http_get = { - host = "localhost"; - port = 4011; - path = "/systems"; - }; - initial_delay_seconds = 2; - period_seconds = 2; - }; + exec = "cd $MOCK_SERVICES_DIR && uv run uvicorn cmdb.app:app --host 0.0.0.0 --port ${toString config.processes.cmdb-mock.ports.http.value}"; + ports.http.allocate = 4011; + ready.http.get = { + host = "localhost"; + port = config.processes.cmdb-mock.ports.http.value; + path = "/systems"; }; }; processes.iam-mock = { - exec = "cd $MOCK_SERVICES_DIR && uv run uvicorn iam.app:app --host 0.0.0.0 --port 4012"; - process-compose = { - readiness_probe = { - http_get = { - host = "localhost"; - port = 4012; - path = "/users"; - }; - initial_delay_seconds = 2; - period_seconds = 2; - }; + exec = "cd $MOCK_SERVICES_DIR && uv run uvicorn iam.app:app --host 0.0.0.0 --port ${toString config.processes.iam-mock.ports.http.value}"; + ports.http.allocate = 4012; + ready.http.get = { + host = "localhost"; + port = config.processes.iam-mock.ports.http.value; + path = "/users"; }; }; } diff --git a/verdict-backend/app/database_config.py b/verdict-backend/app/database_config.py index dfe26f7..9e5dbec 100644 --- a/verdict-backend/app/database_config.py +++ b/verdict-backend/app/database_config.py @@ -1,12 +1,24 @@ +import os from functools import lru_cache from pydantic import Field, SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict +_VALID_ENVIRONMENTS = {"development", "test", "production"} +_environment = os.getenv("ENVIRONMENT", "development") +if _environment not in _VALID_ENVIRONMENTS: + raise ValueError( + f"Invalid ENVIRONMENT {_environment!r}. " + f"Must be one of: {', '.join(sorted(_VALID_ENVIRONMENTS))}" + ) + class DatabaseSettings(BaseSettings): model_config = SettingsConfigDict( - env_file=".env", + env_file=[ + f".env.{_environment}", + f".env.{_environment}.local", + ], env_file_encoding="utf-8", case_sensitive=False, extra="allow", diff --git a/verdict-backend/devenv.nix b/verdict-backend/devenv.nix index 23f55e5..826e4e0 100644 --- a/verdict-backend/devenv.nix +++ b/verdict-backend/devenv.nix @@ -9,19 +9,21 @@ let pythonVersion = lib.fileContents ./.python-version; database_name = "verdict"; postgres_host = "127.0.0.1"; - postgres_port = 5432; postgres_user = "postgres"; postgres_password = "postgres"; in { - process.manager.implementation = "process-compose"; - packages = with pkgs; [ pgcli ]; - # Override mock-services path for the cross-project import. - env.MOCK_SERVICES_DIR = lib.mkForce "${config.devenv.root}/../mock-services"; + env = { + DATABASE_NAME = if config.devenv.isTesting then "${database_name}_test" else database_name; + DATABASE_PORT = toString config.processes.postgres.ports.main.value; + + ENVIRONMENT = if config.devenv.isTesting then "test" else "development"; + VERDICT_URL = "http://localhost:${toString config.processes.verdict-app.ports.http.value}"; + }; languages.python = { enable = true; @@ -36,8 +38,8 @@ in enable = true; package = pkgs.postgresql_16; - listen_addresses = postgres_host; - port = postgres_port; + listen_addresses = lib.mkForce postgres_host; + port = 5432; initialScript = '' DO $$ @@ -75,18 +77,12 @@ in }; processes.verdict-app = { - exec = "uv run uvicorn app.main:app --host 0.0.0.0 --port 8000"; - process-compose = { - disabled = true; - readiness_probe = { - http_get = { - host = "localhost"; - port = 8000; - path = "/"; - }; - initial_delay_seconds = 2; - period_seconds = 2; - }; + ports.http.allocate = 8000; + exec = "uv run uvicorn app.main:app --host 0.0.0.0 --port ${toString config.processes.verdict-app.ports.http.value}"; + ready.http.get = { + host = "localhost"; + port = config.processes.verdict-app.ports.http.value; + path = "/"; }; }; @@ -95,23 +91,6 @@ in ruff-format.enable = true; }; - scripts.gen-env-sample.exec = '' - cat > .env.sample < .env.${environment} </dev/null || true - wait $APP_PID $ASSET_MOCK_PID $CMDB_MOCK_PID $IAM_MOCK_PID 2>/dev/null || true - } - trap cleanup EXIT INT TERM - - # Start mock services - echo "Starting mock asset inventory..." - (cd $MOCK_SERVICES_DIR && exec uv run uvicorn asset_inventory.app:app --host 0.0.0.0 --port 4010) & - ASSET_MOCK_PID=$! - - echo "Starting mock CMDB..." - (cd $MOCK_SERVICES_DIR && exec uv run uvicorn cmdb.app:app --host 0.0.0.0 --port 4011) & - CMDB_MOCK_PID=$! - - echo "Starting mock IAM..." - (cd $MOCK_SERVICES_DIR && exec uv run uvicorn iam.app:app --host 0.0.0.0 --port 4012) & - IAM_MOCK_PID=$! - - retries=0 - until curl -sf http://localhost:4010/assets > /dev/null 2>&1; do - retries=$((retries + 1)) - if [ $retries -ge 30 ]; then - echo "ERROR: Asset inventory mock failed to start" - exit 1 - fi - sleep 1 - done - echo "Asset inventory mock ready" - - retries=0 - until curl -sf http://localhost:4011/systems > /dev/null 2>&1; do - retries=$((retries + 1)) - if [ $retries -ge 30 ]; then - echo "ERROR: CMDB mock failed to start" - exit 1 - fi - sleep 1 - done - echo "CMDB mock ready" - - retries=0 - until curl -sf http://localhost:4012/users > /dev/null 2>&1; do - retries=$((retries + 1)) - if [ $retries -ge 30 ]; then - echo "ERROR: IAM mock failed to start" - exit 1 - fi - sleep 1 - done - echo "IAM mock ready" - - # Start verdict app against test DB - echo "Starting verdict app..." - (DATABASE_NAME=${database_name}_test \ - ASSET_INVENTORY_URL=http://localhost:4010/assets \ - CMDB_URL=http://localhost:4011/systems \ - IAM_URL=http://localhost:4012/users \ - CONFIG_BASEDIR=../config \ - exec uv run uvicorn app.main:app --host 0.0.0.0 --port 8000) & - APP_PID=$! - - retries=0 - until curl -sf http://localhost:8000/ > /dev/null 2>&1; do - retries=$((retries + 1)) - if [ $retries -ge 30 ]; then - echo "ERROR: Verdict app failed to start" - exit 1 - fi - sleep 1 - done - echo "Verdict app ready" - - # Reset test DB and run E2E tests - just db-test-reset - DATABASE_NAME=${database_name}_test just db-migrate - DATABASE_NAME=${database_name}_test uv run pytest --disable-plugin-autoload -p asyncio -m e2e --junit-xml=test-results/e2e.xml + + DATABASE_NAME_TEST=${database_name}_test just test --junit-xml=test-results/unit.xml + + DATABASE_NAME_TEST=${database_name}_test just test-e2e --junit-xml=test-results/e2e.xml ''; } diff --git a/verdict-backend/justfile b/verdict-backend/justfile index 360f3f5..6cd29fd 100644 --- a/verdict-backend/justfile +++ b/verdict-backend/justfile @@ -5,13 +5,6 @@ set dotenv-load default: @just --list -# Start the FastAPI development server -dev: - uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 - -# Start the FastAPI development server against the test database -dev-test: - DATABASE_NAME=$DATABASE_NAME_TEST uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 # Run tests (excludes e2e; use test-e2e for those) test *ARGS: @@ -69,17 +62,6 @@ db-test-reset: db-shell: pgcli -# Show database configuration -db-info: - #!/usr/bin/env bash - echo "Database Configuration:" - echo " DATABASE_URL: $DATABASE_URL" - echo " DATABASE_HOST: $DATABASE_HOST" - echo " DATABASE_PORT: $DATABASE_PORT" - echo " DATABASE_NAME: $DATABASE_NAME" - echo " DATABASE_USER: $DATABASE_USER" - echo " Test DB: $DATABASE_NAME_TEST" - # Regenerate JSON Schema for config/roles/*.yaml from the RoleConfig pydantic model gen-config-schema: uv run python -c "from app.schemas.config import RoleConfig; import json; print(json.dumps(RoleConfig.model_json_schema(), indent=2))" > ../config/roles/schema.json.tmp && mv ../config/roles/schema.json.tmp ../config/roles/schema.json @@ -87,28 +69,3 @@ gen-config-schema: # Create a local user (--role flags are optional and repeatable) create-user username email *ROLES: uv run python -m app.cli --username {{username}} --email {{email}} {{ROLES}} - -# Set up the full development environment -setup: - #!/usr/bin/env bash - echo "Setting up development environment..." - - echo "" - echo "1. Installing Python dependencies..." - uv sync - - echo "" - echo "2. Waiting for PostgreSQL to be ready..." - until pg_isready > /dev/null 2>&1; do - echo "Waiting for PostgreSQL..." - sleep 1 - done - - echo "" - echo "3. Running database migrations..." - uv run alembic upgrade head - - echo "" - echo "✅ Setup complete!" - echo "" - just --list diff --git a/verdict-backend/tests/e2e/conftest.py b/verdict-backend/tests/e2e/conftest.py index 5c4d0c7..569b329 100644 --- a/verdict-backend/tests/e2e/conftest.py +++ b/verdict-backend/tests/e2e/conftest.py @@ -1,5 +1,6 @@ """Fixtures for E2E tests that run against real services.""" +import os from collections.abc import Generator import httpx @@ -28,7 +29,7 @@ def _truncate() -> None: @pytest.fixture(scope="session") def base_url() -> str: - return "http://localhost:8000" + return os.environ.get("VERDICT_URL", "http://localhost:8000") @pytest.fixture(scope="session") diff --git a/verdict-backend/tests/routes/test_ingestion.py b/verdict-backend/tests/routes/test_ingestion.py index 333be6f..a888032 100644 --- a/verdict-backend/tests/routes/test_ingestion.py +++ b/verdict-backend/tests/routes/test_ingestion.py @@ -6,6 +6,7 @@ from httpx import AsyncClient from sqlmodel import Session, select +from app.config import settings from app.deps import get_http_client from app.main import app from app.models.system import System @@ -47,12 +48,12 @@ def _override() -> Generator[httpx.Client, None, None]: def _mock_all_services() -> None: - respx.get("http://localhost:4010/assets").respond(200, json=ASSET_INDEX) + respx.get(settings.asset_inventory_url).respond(200, json=ASSET_INDEX) for item_id, payload in ASSET_DETAIL.items(): - respx.get(f"http://localhost:4010/assets/{item_id}").respond(200, json=payload) - respx.get("http://localhost:4011/systems").respond(200, json=SYSTEM_INDEX) + respx.get(f"{settings.asset_inventory_url}/{item_id}").respond(200, json=payload) + respx.get(settings.cmdb_url).respond(200, json=SYSTEM_INDEX) for item_id, payload in SYSTEM_DETAIL.items(): - respx.get(f"http://localhost:4011/systems/{item_id}").respond(200, json=payload) + respx.get(f"{settings.cmdb_url}/{item_id}").respond(200, json=payload) @respx.mock @@ -68,12 +69,12 @@ async def test_full_ingest_endpoint(app_client: AsyncClient, db_session, mock_ht def _mock_system_failure() -> None: - respx.get("http://localhost:4010/assets").respond(200, json=ASSET_INDEX) + respx.get(settings.asset_inventory_url).respond(200, json=ASSET_INDEX) for item_id, payload in ASSET_DETAIL.items(): - respx.get(f"http://localhost:4010/assets/{item_id}").respond(200, json=payload) - respx.get("http://localhost:4011/systems").respond(200, json=SYSTEM_INDEX) + respx.get(f"{settings.asset_inventory_url}/{item_id}").respond(200, json=payload) + respx.get(settings.cmdb_url).respond(200, json=SYSTEM_INDEX) # System detail references a nonexistent asset - respx.get("http://localhost:4011/systems/SYS-001").respond( + respx.get(f"{settings.cmdb_url}/SYS-001").respond( 200, json={ "id": "SYS-001", From f58633d0e31ab615fbaf06d0056532ee8ed76bf7 Mon Sep 17 00:00:00 2001 From: Christian Schwartz Date: Fri, 20 Mar 2026 17:21:23 +0100 Subject: [PATCH 2/3] Implemented "Ignore .env.production too." --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0b974f2..424270d 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,8 @@ devenv.local.nix # generated by devenv .env.development .env.test -# local overrides (never commit) +# production secrets and local overrides (never commit) +.env.production .env.*.local # AI tooling From 32ea4e6483fb92c212145af0f775d4cff527f35e Mon Sep 17 00:00:00 2001 From: Christian Schwartz Date: Fri, 20 Mar 2026 17:27:44 +0100 Subject: [PATCH 3/3] Implemented "Add explicit Postgres process dependency to database tasks." --- verdict-backend/devenv.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/verdict-backend/devenv.nix b/verdict-backend/devenv.nix index 826e4e0..2596488 100644 --- a/verdict-backend/devenv.nix +++ b/verdict-backend/devenv.nix @@ -131,7 +131,7 @@ in tasks."verdict-app:db-migrate" = lib.mkIf (!config.devenv.isTesting) { exec = "just db-migrate"; - after = [ "verdict-app:gen-env" ]; + after = [ "verdict-app:gen-env" "devenv:processes:postgres" ]; before = [ "devenv:processes:verdict-app" ]; }; @@ -143,7 +143,7 @@ in just db-migrate echo "db-setup: done" ''; - after = [ "verdict-app:gen-env" ]; + after = [ "verdict-app:gen-env" "devenv:processes:postgres" ]; before = [ "devenv:processes:verdict-app" ]; };