diff --git a/.github/workflows/backend_build_check.yml b/.github/workflows/backend_build_check.yml new file mode 100644 index 000000000..e45e9db5e --- /dev/null +++ b/.github/workflows/backend_build_check.yml @@ -0,0 +1,20 @@ +name: Build Docker Images +on: + pull_request: + branches: + - master + - develop + paths: + - "backend/**" + - ".github/workflows/docker_build.yml" + +jobs: + # Backend API image. amd64-only (GDAL + source-built tippecanoe make arm64 + # emulation slow). Build-only check for PRs + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@4.0.3 + with: + context: backend/ + dockerfile: Dockerfile + image_name: ghcr.io/${{ github.repository_owner }}/fair/api + push: false diff --git a/.github/workflows/deploy_frontend_s3.yml b/.github/workflows/deploy_frontend_s3.yml new file mode 100644 index 000000000..68df7d0bf --- /dev/null +++ b/.github/workflows/deploy_frontend_s3.yml @@ -0,0 +1,82 @@ +name: Deploy Production Frontend to S3 & CloudFront + +on: + push: + branches: + - main + paths: + - 'frontend/**' + - '.github/workflows/deploy_frontend_s3.yml' + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./frontend + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install pnpm + uses: pnpm/action-setup@v3 + with: + version: 9.8.0 + + - name: Get pnpm store directory + id: pnpm-cache + run: echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install Dependencies + run: pnpm install --frozen-lockfile + + - name: Build Static Production Assets + env: + VITE_BASE_API_URL: "https://api.fair.hotosm.org/api/v1/" + VITE_NODE_ENV: "production" + VITE_FAIR_PROD_URL: "https://fair.hotosm.org/" + VITE_HANKO_URL: "https://login.hotosm.org" + run: pnpm run build + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::670261699094:role/Github-AWS-OIDC + aws-region: us-east-1 + + - name: Sync Static Assets to S3 + run: | + aws s3 sync dist/ s3://hotosm-fair-frontend-production \ + --delete \ + --cache-control "public, max-age=31536000, immutable" \ + --exclude "index.html" + + # Sync index.html with no-cache so client browsers instantly receive app updates + aws s3 cp dist/index.html s3://hotosm-fair-frontend-production/index.html \ + --cache-control "no-cache, no-store, must-revalidate" + + - name: Invalidate CloudFront CDN Cache + run: | + DISTRIBUTION_ID=$(aws cloudfront list-distributions --query "DistributionList.Items[?Aliases.Items && contains(Aliases.Items, 'fair.hotosm.org')].Id | [0]" --output text) + if [ "$DISTRIBUTION_ID" != "None" ] && [ -n "$DISTRIBUTION_ID" ]; then + aws cloudfront create-invalidation --distribution-id "$DISTRIBUTION_ID" --paths "/*" + fi \ No newline at end of file diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml deleted file mode 100644 index b0a9c877b..000000000 --- a/.github/workflows/docker_build.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Build Docker Images -on: - push: - branches: - - master - - develop - paths: - - "backend/**" - - ".github/workflows/docker_build.yml" - pull_request: - branches: - - master - - develop - paths: - - "backend/**" - - ".github/workflows/docker_build.yml" - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-api-image: - runs-on: ubuntu-24.04 - permissions: - contents: read - packages: write - steps: - - name: Remove unnecessary files - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - uses: actions/checkout@v5 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for API Docker - id: meta_api - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-api - tags: | - type=ref,event=branch - type=ref,event=tag - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=test - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build API Docker image - uses: docker/build-push-action@v6 - with: - context: backend/ - file: backend/Dockerfile - platforms: linux/amd64 - provenance: false - sbom: false - push: true - tags: ${{ steps.meta_api.outputs.tags }} - labels: ${{ steps.meta_api.outputs.labels }} - cache-from: type=gha,scope=api,timeout=20m - cache-to: type=gha,mode=max,scope=api,timeout=20m,ignore-error=true - github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker_publish_image.yml b/.github/workflows/docker_publish_image.yml deleted file mode 100644 index 536befb85..000000000 --- a/.github/workflows/docker_publish_image.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Build and Publish Docker Images - -on: - push: - branches: - - master - - develop - paths-ignore: - - ".github/workflows/backend_build.yml" - - ".github/workflows/frontend_build.yml" - - ".github/workflows/frontend_build_push.yml" - release: - types: [released] - workflow_dispatch: - inputs: - use_cache: - description: "Use Docker build cache" - required: false - default: "true" - type: choice - options: - - "true" - - "false" - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-and-push-api-image: - runs-on: ubuntu-24.04 - permissions: - contents: read - packages: write - steps: - - name: Remove unnecessary files - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - uses: actions/checkout@v5 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for API Docker - id: meta_api - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-api - tags: | - type=ref,event=branch - type=ref,event=tag - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push API Docker image - id: build_api - uses: docker/build-push-action@v6 - with: - context: backend/ - file: backend/Dockerfile - platforms: linux/amd64 - provenance: false - sbom: false - push: true - tags: ${{ steps.meta_api.outputs.tags }} - labels: ${{ steps.meta_api.outputs.labels }} - cache-from: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_cache == 'false') && '' || 'type=gha,scope=api,timeout=20m' }} - cache-to: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.use_cache == 'false') && '' || 'type=gha,mode=max,scope=api,timeout=20m,ignore-error=true' }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Validate API manifest - env: - IMAGE_REF: ${{ env.REGISTRY }}/${{ github.repository_owner }}/fair-api@${{ steps.build_api.outputs.digest }} - run: | - json=$(docker buildx imagetools inspect --format '{{json .}}' "$IMAGE_REF") - echo "$json" | jq -e ' - if (.manifest.manifests? | type) == "array" then - (any(.manifest.manifests[]?; .platform.os == "linux" and .platform.architecture == "amd64")) - and - (all(.manifest.manifests[]?; (.platform.os != "unknown" and .platform.architecture != "unknown"))) - else - (.image.os == "linux" and .image.architecture == "amd64") - end - ' >/dev/null diff --git a/.github/workflows/frontend_build.yml b/.github/workflows/frontend_build_check.yml similarity index 63% rename from .github/workflows/frontend_build.yml rename to .github/workflows/frontend_build_check.yml index 3066f2198..a8c34dba8 100644 --- a/.github/workflows/frontend_build.yml +++ b/.github/workflows/frontend_build_check.yml @@ -14,7 +14,7 @@ on: jobs: frontend-unit-test: - uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@3.2.0 + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@4.0.3 with: working_dir: frontend Build_On_Ubuntu: @@ -54,31 +54,22 @@ jobs: working-directory: ./frontend run: pnpm run build - Docker_Build: - runs-on: ubuntu-latest + # Build-only checks for PRs. Push happens in docker_publish_image.yml on + # merge to master/stage/develop. + docker-build-dev: needs: [frontend-unit-test] + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@4.0.3 + with: + context: frontend/ + dockerfile: Dockerfile.dev + image_name: ghcr.io/${{ github.repository_owner }}/fair/frontend + push: false - steps: - - name: Checkout repository - uses: actions/checkout@v5 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build dev image - uses: docker/build-push-action@v6 - with: - context: ./frontend - file: ./frontend/Dockerfile.dev - push: false - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build prod image - uses: docker/build-push-action@v6 - with: - context: ./frontend - file: ./frontend/Dockerfile.prod - push: false - cache-from: type=gha - cache-to: type=gha,mode=max + docker-build-prod: + needs: [frontend-unit-test] + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@4.0.3 + with: + context: frontend/ + dockerfile: Dockerfile.prod + image_name: ghcr.io/${{ github.repository_owner }}/fair/frontend + push: false diff --git a/.github/workflows/frontend_build_push.yml b/.github/workflows/frontend_build_push.yml deleted file mode 100644 index a6ae72e22..000000000 --- a/.github/workflows/frontend_build_push.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Frontend Build and upload to S3 - -on: - release: - types: [released] - workflow_dispatch: - -permissions: - id-token: write - contents: read - -jobs: - frontend-unit-test: - uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@3.2.0 - - with: - working_dir: frontend - build_and_upload: - runs-on: ubuntu-latest - needs: [frontend-unit-test] - environment: Production - - env: - CI: false - - steps: - - name: Check out Git repository - uses: actions/checkout@v5 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - - name: Install pnpm - run: npm install -g pnpm - - - name: Cache pnpm store - uses: actions/cache@v4 - id: pnpm-cache - with: - path: ~/.pnpm-store - key: ${{ runner.os }}-pnpm-${{ hashFiles('frontend/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - - name: Install Node.js dependencies - if: steps.pnpm-cache.outputs.cache-hit != 'true' - working-directory: ./frontend - run: pnpm install --frozen-lockfile - - - name: Build frontend - working-directory: ./frontend - run: pnpm run build - env: - VITE_BASE_API_URL: ${{ vars.VITE_BASE_API_URL }} - VITE_MATOMO_ID: ${{ vars.VITE_MATOMO_ID }} - VITE_MATOMO_APP_DOMAIN: ${{ vars.VITE_MATOMO_APP_DOMAIN }} - VITE_OSM_HASHTAGS: ${{ vars.VITE_OSM_HASHTAGS }} - VITE_FAIR_PREDICTOR_API_URL: ${{ vars.VITE_FAIR_PREDICTOR_API_URL }} - # VITE_MIN_TRAINING_AREA_SIZE: ${{ vars.VITE_MIN_TRAINING_AREA_SIZE }} - - - name: Authenticate to AWS - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-east-1 - role-to-assume: ${{ secrets.AWS_OIDC_ROLE }} - role-session-name: fAIrGithub - - - name: Upload to S3 - working-directory: ./frontend/dist - run: aws s3 sync . s3://${{ vars.FRONTEND_BUCKET }}/ diff --git a/.github/workflows/publish_container_image.yml b/.github/workflows/publish_container_image.yml new file mode 100644 index 000000000..845aceefe --- /dev/null +++ b/.github/workflows/publish_container_image.yml @@ -0,0 +1,32 @@ +name: Build and Publish Docker Images + +on: + push: + branches: + - master + - stage + - develop + paths-ignore: + - ".github/workflows/backend_build.yml" + - ".github/workflows/frontend_build.yml" + - ".github/workflows/frontend_build_push.yml" + release: + types: [released] + workflow_dispatch: + +jobs: + # Backend API image. amd64-only (GDAL + source-built tippecanoe make arm64 + # emulation slow) + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@4.0.3 + with: + context: backend/ + dockerfile: Dockerfile + image_name: ghcr.io/${{ github.repository_owner }}/fair/api + + frontend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@4.0.3 + with: + context: frontend/ + dockerfile: Dockerfile.prod + image_name: ghcr.io/${{ github.repository_owner }}/fair/frontend diff --git a/.github/workflows/release_chart.yaml b/.github/workflows/release_chart.yaml index 3222e1b39..da4444c3e 100755 --- a/.github/workflows/release_chart.yaml +++ b/.github/workflows/release_chart.yaml @@ -17,7 +17,7 @@ permissions: jobs: publish: - uses: hotosm/gh-workflows/.github/workflows/just.yml@3.3.2 + uses: hotosm/gh-workflows/.github/workflows/just.yml@4.0.3 with: environment: "test" command: "chart publish" diff --git a/.github/workflows/zenml_postgres_build.yml b/.github/workflows/zenml_postgres_build.yml index 8270a1a04..8c56980ec 100644 --- a/.github/workflows/zenml_postgres_build.yml +++ b/.github/workflows/zenml_postgres_build.yml @@ -10,22 +10,20 @@ on: jobs: server-image: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@3.6.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@4.0.3 with: context: infra/zenml - # This remains hotosm/zenml-postgres for legacy reasons - # Ideally it would be hotosm/fair/zenml-postgres - image_name: ghcr.io/${{ github.repository_owner }}/zenml-postgres + image_name: ghcr.io/${{ github.repository_owner }}/fair/zenml-postgres build_target: runtime dockerfile: Dockerfile.postgres extra_build_args: ZENML_VERSION=${{ inputs.zenml_version }} - image_tags: ghcr.io/${{ github.repository_owner }}/zenml-postgres:${{ inputs.zenml_version }} + image_tags: ghcr.io/${{ github.repository_owner }}/fair/zenml-postgres:${{ inputs.zenml_version }} # Workaround until CVE-2026-27143 fixed in ZenML upstream image... scan_image: false cli-image: needs: server-image - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@3.6.0 + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@4.0.3 with: context: infra/zenml image_name: ghcr.io/${{ github.repository_owner }}/fair/cli diff --git a/.gitignore b/.gitignore index 94512cda9..19a4f75e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ runs +CLAUDE.md # frontend # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. @@ -68,4 +69,4 @@ fair-app-data/* # helm charts -fair-*.tgz +fair-*.tgz \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d2c43de7..547d04d70 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -default_install_hook_types: [ pre-commit, commit-msg, pre-push ] +default_install_hook_types: [pre-commit, commit-msg, pre-push] default_language_version: python: python3.12 @@ -7,7 +7,7 @@ repos: rev: v0.15.12 hooks: - id: ruff-check - args: [ --fix ] + args: [--fix] files: ^backend/ - id: ruff-format files: ^backend/ @@ -18,7 +18,7 @@ repos: name: ty entry: uv run --directory backend ty check language: system - types: [ python ] + types: [python] files: ^backend/ pass_filenames: false @@ -33,14 +33,13 @@ repos: name: pytest (backend) entry: uv run --directory backend pytest language: system - types: [ python ] + types: [python] files: ^backend/ pass_filenames: false - stages: [ pre-push ] + stages: [pre-push] - id: commitizen name: commitizen check - entry: uv run cz--config pyproject.toml check --commit-msg-file - files: ^backend/ + entry: uv run --project backend cz --config backend/pyproject.toml check --commit-msg-file language: system - stages: [ commit-msg ] + stages: [commit-msg] diff --git a/backend/README.md b/backend/README.md index 8ee5a22cc..52b07e5bf 100644 --- a/backend/README.md +++ b/backend/README.md @@ -2,20 +2,33 @@ Thin coordination layer for the fAIr AI-Assisted Mapping platform. Owns the public REST API, the user database, and the orchestration of dataset builds, training runs and predictions. ML pipelines and STAC catalog operations live in [fair-py-ops](https://github.com/hotosm/fAIr-models) and run on a ZenML stack. -Model code lives in per-model docker images that the ZenML k8s orchestrator pulls. +Model code lives in per-model docker images that the ZenML orchestrator pulls. ## Quick start -Prerequisites: a running fair-py-ops dev stack (kind cluster + ZenML + STAC API + MinIO + MLflow). See [fair-py-ops/infra/cli](https://github.com/hotosm/fAIr-models/tree/main/infra/cli) for the single-image bring-up. +The compose file at the repository root runs this backend and everything it +depends on. See [docs/Docker-installation.md](../docs/Docker-installation.md). + +### Running the backend on the host + +Useful when iterating on backend code. Start the dependencies in docker, then +run Django outside it: ```bash -just setup # uv sync + pre-commit install -cp env_example .env # fill in real values +cd .. +docker compose up -d postgres minio stac mlflow zenml +cd backend +just setup +cp env_example .env just migrate -just run # dev server on :8000 -just worker # second terminal: db_worker for background tasks +just run +just worker tasks ``` +The two sample files differ only in host names: the root `env_example` uses +compose service names (`postgres`, `minio`, `stac`, `zenml`), while +`backend/env_example` uses `localhost` with the ports those services publish. + OpenAPI schema at `/api/schema/`, Swagger UI at `/api/docs/`, ReDoc at `/api/redoc/`. ## Environment @@ -60,12 +73,19 @@ OpenAPI schema at `/api/schema/`, Swagger UI at `/api/docs/`, ReDoc at `/api/red ### fair-py-ops (ZenML + STAC) +`FAIR_*` values are read by this backend. The `ZENML_STORE_*` values are read by +the `zenml` library itself when it opens a connection, so both sets point at the +same server. Authenticate with either an API key or a username and password. + | Name | Required | Default | Description | |------|----------|---------|-------------| | `FAIR_ZENML_STORE_URL` | yes (at runtime) | `null` | URL of the deployed ZenML server. Optional at boot, raises loud at first call site. | -| `FAIR_ZENML_STORE_API_KEY` | yes (at runtime) | `null` | Mint via `kubectl exec deploy/zenml -- zenml service-account create fair-cli`. | -| `FAIR_STAC_API_URL` | yes (at runtime) | `null` | URL of the STAC API root (eoapi-stac-fastapi). | +| `FAIR_STAC_API_URL` | yes (at runtime) | `null` | URL of the STAC API root (eoapi-stac-fastapi). Trailing slashes are stripped. | | `FAIR_STAC_API_KEY` | prod | `null` | Bearer token for the STAC Transactions extension. | +| `ZENML_STORE_URL` | yes (at runtime) | `null` | Same server as `FAIR_ZENML_STORE_URL`. | +| `ZENML_STORE_API_KEY` | one of the two | `null` | Service-account key. Mint with `zenml service-account create fair-backend`. | +| `ZENML_STORE_USERNAME` | one of the two | `null` | Username, paired with `ZENML_STORE_PASSWORD`. The compose stack's default user is `default` with an empty password. | +| `ZENML_STORE_PASSWORD` | with username | `null` | Password for the above. | ### Object storage (S3 / MinIO) @@ -172,4 +192,4 @@ just test # pytest just lint # pre-commit run --all-files (ruff + format + ty + uv-lock-check + commitizen) ``` -Tests under `backend/tests/` import across apps and mock `shared.integrations.zenml`; nothing hits a live ZenML server. For end-to-end checks against the dev stack drive the public API with curl (see `/api/docs/`). +Tests under `backend/tests/` import across apps and mock `shared.integrations.zenml`; nothing hits a live ZenML server. For end-to-end checks against a running stack, run [`test.py`](../test.py), which drives the public API end to end. diff --git a/backend/config/env.py b/backend/config/env.py index 9464cc33e..a2e1e1a52 100644 --- a/backend/config/env.py +++ b/backend/config/env.py @@ -54,6 +54,7 @@ def _check_database_url(cls, value: str) -> str: allowed_hosts: Annotated[list[str], NoDecode] = Field(default_factory=list) csrf_trusted_origins: Annotated[list[str], NoDecode] = Field(default_factory=list) cors_allowed_origins: Annotated[list[str], NoDecode] = Field(default_factory=list) + cors_allow_all_origins: bool = False secure_ssl_redirect: bool = True # Frontend / API URLs @@ -61,6 +62,13 @@ def _check_database_url(cls, value: str) -> str: api_base_url: AnyHttpUrl hostname: str = "127.0.0.1" + # Serve the built frontend SPA from this backend (bundled deploy). When true, + # WhiteNoise serves the SPA assets from `frontend_dist_dir` and a catch-all + # route returns index.html for client-side routes. Leave false when the + # frontend is served separately (e.g. S3 + CloudFront). + serve_frontend: bool = False + frontend_dist_dir: Path = BASE_DIR / "frontend_html" + # Authentication auth_provider: AuthProvider = AuthProvider.HANKO fair_dev_token: SecretStr | None = None diff --git a/backend/config/settings.py b/backend/config/settings.py index f6f22e596..9607d5937 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -66,7 +66,11 @@ def _str(value: Any | None) -> str | None: FAIR_ZENML_STORE_URL = _str(settings.fair_zenml_store_url) FAIR_ZENML_STORE_API_KEY = _secret(settings.fair_zenml_store_api_key) -FAIR_STAC_API_URL = _str(settings.fair_stac_api_url) +# AnyHttpUrl appends a trailing slash when the URL carries no path, which would +# double up against the "/collections/..." suffixes appended at every call site. +FAIR_STAC_API_URL = ( + _str(settings.fair_stac_api_url).rstrip("/") if settings.fair_stac_api_url else None +) FAIR_STAC_API_KEY = _secret(settings.fair_stac_api_key) BUCKET_NAME = settings.bucket_name @@ -121,6 +125,8 @@ def _str(value: Any | None) -> str | None: MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", "django.middleware.security.SecurityMiddleware", + # Optional deployment of frontend dist from the server + "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -250,6 +256,12 @@ def _str(value: Any | None) -> str | None: STATIC_ROOT = str(BASE_DIR / "api_static") MEDIA_ROOT = str(BASE_DIR / "media") +SERVE_FRONTEND = settings.serve_frontend +FRONTEND_DIST_DIR = settings.frontend_dist_dir +if SERVE_FRONTEND: + WHITENOISE_ROOT = str(FRONTEND_DIST_DIR) + WHITENOISE_INDEX_FILE = True + _logger_handlers: list[str] = ["console"] if DEBUG else ["console", "file"] _log_handlers: dict[str, Any] = { "console": { @@ -344,6 +356,13 @@ def _str(value: Any | None) -> str | None: "COMPONENT_SPLIT_REQUEST": True, "SCHEMA_PATH_PREFIX": "/api/v1", "SCHEMA_PATH_PREFIX_TRIM": True, + "ENUM_NAME_OVERRIDES": { + "PipelineRunStatus": "shared.enums.PipelineRunStatus.choices", + "DatasetStatus": "shared.enums.DatasetStatus.choices", + "LocalModelStatus": "shared.enums.LocalModelStatus.choices", + "BaseModelStatus": "shared.enums.BaseModelStatus.choices", + "ModelCategory": "shared.enums.ModelCategory.choices", + }, } TEST_RUNNER = "tests.test_runners.NoDestroyTestRunner" @@ -365,9 +384,9 @@ def _extract_domain(url: str) -> str | None: return urlparse(url).hostname -if DEBUG: +if DEBUG or settings.cors_allow_all_origins: CORS_ALLOW_ALL_ORIGINS = True - CORS_ALLOW_CREDENTIALS = False + CORS_ALLOW_CREDENTIALS = True else: CORS_ALLOW_ALL_ORIGINS = False CORS_ALLOW_CREDENTIALS = True diff --git a/backend/config/urls.py b/backend/config/urls.py index 8b1971196..3bcd63e1b 100644 --- a/backend/config/urls.py +++ b/backend/config/urls.py @@ -72,3 +72,32 @@ def home(_request): urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + + +if settings.SERVE_FRONTEND: + from django.http import Http404, HttpResponse + from django.urls import re_path + + def spa_index(_request): + """Return the bundled SPA shell for any non-API route (client-side routing). + + Intercepts all unmatched routes. + (Everything except the API, Django admin, and static/media) + """ + index_file = Path(settings.FRONTEND_DIST_DIR) / "index.html" + try: + return HttpResponse(index_file.read_bytes(), content_type="text/html") + except FileNotFoundError as exc: + raise Http404("Frontend bundle not found") from exc + + urlpatterns += [ + re_path( + # We handle all URLs other than the API / Static / Admin ones + # in the frontend. This also accounts for trailing slashes. + # Groups stay non-capturing: re_path forwards captured groups as + # positional view arguments, which spa_index does not accept. + r"^(?!api(?:/|$)|api_static/|media(?:/|$)|django-admin(?:/|$)).*$", + spa_index, + name="spa", + ), + ] diff --git a/backend/datasets/migrations/0004_dataset_dataset_status_valid_and_more.py b/backend/datasets/migrations/0004_dataset_dataset_status_valid_and_more.py new file mode 100644 index 000000000..dabddc2ab --- /dev/null +++ b/backend/datasets/migrations/0004_dataset_dataset_status_valid_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.4 on 2026-07-22 11:40 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('datasets', '0003_remove_dataset_datasets_da_build_s_7fd7fa_idx_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddConstraint( + model_name='dataset', + constraint=models.CheckConstraint(condition=models.Q(('status__in', ['draft', 'building', 'built', 'failed'])), name='dataset_status_valid'), + ), + migrations.AddConstraint( + model_name='dataset', + constraint=models.CheckConstraint(condition=models.Q(('visibility__in', ['private', 'public'])), name='dataset_visibility_valid'), + ), + ] diff --git a/backend/datasets/models.py b/backend/datasets/models.py index 51955dcba..897fa1c28 100644 --- a/backend/datasets/models.py +++ b/backend/datasets/models.py @@ -2,21 +2,20 @@ from django.db import models, transaction from accounts.models import OsmUser -from shared.enums import Visibility +from shared.enums import DatasetStatus, Visibility from shared.validators import validate_geometry class Dataset(models.Model): - class Status(models.TextChoices): - DRAFT = "draft", "Draft" - BUILDING = "building", "Building" - BUILT = "built", "Built" - FAILED = "failed", "Failed" + # Module-level enum: a nested class body cannot see it from inside Meta. + Status = DatasetStatus stac_id = models.CharField(max_length=200, unique=True) title = models.CharField(max_length=200) source_imagery = models.URLField() - status = models.CharField(max_length=20, choices=Status.choices, default=Status.DRAFT) + status = models.CharField( + max_length=20, choices=DatasetStatus.choices, default=DatasetStatus.DRAFT + ) visibility = models.CharField( max_length=20, choices=Visibility.choices, default=Visibility.PRIVATE, db_index=True ) @@ -31,6 +30,16 @@ class Meta: models.Index(fields=["status"]), models.Index(fields=["user"]), ] + constraints = [ + models.CheckConstraint( + condition=models.Q(status__in=DatasetStatus.values), + name="dataset_status_valid", + ), + models.CheckConstraint( + condition=models.Q(visibility__in=Visibility.values), + name="dataset_visibility_valid", + ), + ] ordering = ["-created_at"] def __str__(self) -> str: diff --git a/backend/datasets/views.py b/backend/datasets/views.py index 53ee9def9..03d2b56ca 100644 --- a/backend/datasets/views.py +++ b/backend/datasets/views.py @@ -6,7 +6,12 @@ from django.utils.text import slugify from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view +from drf_spectacular.utils import ( + OpenApiExample, + OpenApiResponse, + extend_schema, + extend_schema_view, +) from gpxpy.gpx import GPX, GPXTrack, GPXTrackPoint, GPXTrackSegment from rest_framework import filters, status, viewsets from rest_framework.decorators import action @@ -155,7 +160,31 @@ def pin(self, request, pk: int | None = None) -> Response: ) return Response(serializer.data, status=status.HTTP_200_OK) - @extend_schema(request=DatasetCreateSerializer, responses={202: DatasetSerializer}) + @extend_schema( + request=DatasetCreateSerializer, + responses={202: DatasetSerializer}, + examples=[ + OpenApiExample( + "Buildings dataset", + value={ + "title": "banepa-buildings", + "description": "Buildings training dataset", + "source_imagery": ( + "https://tiles.openaerialmap.org/62d85d11d8499800053796c1/0/" + "62d85d11d8499800053796c2/{z}/{x}/{y}" + ), + "zoom": 19, + "aoi_ids": [1], + "label_tasks": ["semantic-segmentation"], + "label_classes": [{"name": "building", "classes": ["*"]}], + "keywords": ["building", "polygon"], + "label_type": "vector", + "geometry_type": "polygon", + }, + request_only=True, + ) + ], + ) @action(detail=False, methods=["post"], url_path="build") def build(self, request) -> Response: serializer = DatasetCreateSerializer(data=request.data) @@ -193,7 +222,31 @@ def build(self, request) -> Response: @extend_schema_view( list=extend_schema(description="List AOIs visible to the caller, optionally filtered by bbox."), - create=extend_schema(description="Create an AOI polygon owned by the caller."), + create=extend_schema( + description="Create an AOI polygon owned by the caller.", + examples=[ + OpenApiExample( + "Banepa AOI", + value={ + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [85.51678, 27.63133], + [85.52323, 27.63133], + [85.52323, 27.63743], + [85.51678, 27.63743], + [85.51678, 27.63133], + ] + ], + }, + "properties": {"dataset": None}, + }, + request_only=True, + ) + ], + ), retrieve=extend_schema(description="Retrieve one AOI by id."), update=extend_schema(description="Replace an AOI (owner/admin only)."), partial_update=extend_schema(description="Patch an AOI (owner/admin only)."), diff --git a/backend/env_example b/backend/env_example index fea38a6ed..6fbe47bd6 100644 --- a/backend/env_example +++ b/backend/env_example @@ -1,22 +1,24 @@ -# Required for the backend to boot. Missing values raise at startup, no fallbacks. -# Copy to .env and fill in real values. +# For running Django directly on the host against the compose dependencies: +# +# docker compose up -d postgres minio stac mlflow zenml +# Missing values raise at startup, no fallbacks. Copy to .env and fill in. # Django DEBUG=true SECRET_KEY=change-me-min-32-chars-long-please-please -DATABASE_URL=postgis://admin:password@localhost:5432/fair +DATABASE_URL=postgis://admin:password@localhost:5434/fair ALLOWED_HOSTS=localhost,127.0.0.1 FRONTEND_URL=http://localhost:3500 API_BASE_URL=http://localhost:8000/api/v1 # Authentication provider: hanko (production / SSO) or dev (local-only static token) -AUTH_PROVIDER=hanko +AUTH_PROVIDER=dev # DEV provider (required when AUTH_PROVIDER=dev). Anyone with this token has # full dev-user access; keep in .env, never commit. Generate with `openssl rand -hex 32`. -FAIR_DEV_TOKEN= +FAIR_DEV_TOKEN=dev-token -# Hanko SSO (required when AUTH_PROVIDER=hanko) +# Hanko SSO (required when AUTH_PROVIDER=hanko; both values must be set) HANKO_API_URL= COOKIE_SECRET= COOKIE_DOMAIN= @@ -27,18 +29,24 @@ LOGIN_BACKEND_URL= # Only required when Hanko's "connect existing OSM account" flow is in use. OSM_LOGIN_REDIRECT_URI= -# fair-py-ops connection (hosted ZenML + hosted STAC API) -FAIR_ZENML_STORE_URL= -FAIR_ZENML_STORE_API_KEY= -FAIR_STAC_API_URL= +# fair-py-ops connection. FAIR_* is read by this backend, ZENML_STORE_* by the +# zenml library. Both point at the same server. The compose stack's default user +# is `default` with an empty password; set ZENML_STORE_API_KEY instead for a +# hosted deployment. +FAIR_ZENML_STORE_URL=http://localhost:8080 +FAIR_STAC_API_URL=http://localhost:8082 FAIR_STAC_API_KEY= +ZENML_STORE_URL=http://localhost:8080 +ZENML_STORE_USERNAME=default +ZENML_STORE_PASSWORD= # S3 / object storage -BUCKET_NAME= +BUCKET_NAME=fair-data PARENT_BUCKET_FOLDER=dev AWS_REGION=us-east-1 -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= +AWS_ENDPOINT_URL=http://localhost:9000 +AWS_ACCESS_KEY_ID=minioadmin +AWS_SECRET_ACCESS_KEY=minioadmin PRESIGNED_URL_EXPIRY=900 # Rate limits (used by DRF throttling) diff --git a/backend/feedback/models.py b/backend/feedback/models.py index 3f3628b49..8dd6bb0d4 100644 --- a/backend/feedback/models.py +++ b/backend/feedback/models.py @@ -2,17 +2,18 @@ from django.db import models from accounts.models import OsmUser +from shared.enums import FeedbackAction from shared.validators import validate_geometry class Feedback(models.Model): - class Action(models.TextChoices): - ACCEPT = "accept", "Accept" - REJECT = "reject", "Reject" + Action = FeedbackAction stac_id = models.CharField(max_length=200, db_index=True) geom = geomodels.GeometryField(srid=4326) - action = models.CharField(max_length=6, choices=Action.choices, default=Action.ACCEPT) + action = models.CharField( + max_length=6, choices=FeedbackAction.choices, default=FeedbackAction.ACCEPT + ) comments = models.TextField(blank=True) config = models.JSONField(default=dict, blank=True) user = models.ForeignKey( diff --git a/backend/modelregistry/admin.py b/backend/modelregistry/admin.py index def0dee3d..9c4f803e7 100644 --- a/backend/modelregistry/admin.py +++ b/backend/modelregistry/admin.py @@ -1,14 +1,46 @@ -from django.contrib import admin +from django.contrib import admin, messages -from .models import LocalModel +from .models import BaseModel, LocalModel @admin.register(LocalModel) class LocalModelAdmin(admin.ModelAdmin): - list_display = ["name", "status", "user", "created_at"] - list_filter = ["status", "created_at"] + list_display = ["name", "category", "status", "user", "created_at"] + list_editable = ["category", "status"] + list_filter = ["status", "category", "created_at"] search_fields = ["name", "user__username"] readonly_fields = ["created_at", "last_modified"] date_hierarchy = "created_at" list_per_page = 50 autocomplete_fields = ["user"] + + +@admin.register(BaseModel) +class BaseModelAdmin(admin.ModelAdmin): + list_display = ["name", "category", "status", "visibility", "user", "created_at"] + list_editable = ["category", "status", "visibility"] + list_filter = ["status", "category", "visibility", "created_at"] + search_fields = ["name", "user__username"] + readonly_fields = ["created_at", "last_modified"] + date_hierarchy = "created_at" + list_per_page = 50 + autocomplete_fields = ["user"] + actions = ["register_in_stac"] + + @admin.action(description="Register / re-register selected in STAC") + def register_in_stac(self, request, queryset) -> None: + from .tasks import register_base_model + + enqueued = 0 + for base_model in queryset: + if not (base_model.stac_item or base_model.stac_item_url): + messages.warning( + request, f"{base_model.name}: no stac_item or stac_item_url stored, skipped" + ) + continue + base_model.status = BaseModel.Status.REGISTERING + base_model.error = "" + base_model.save(update_fields=["status", "error", "last_modified"]) + register_base_model.enqueue(base_model_id=base_model.id) + enqueued += 1 + messages.info(request, f"Enqueued {enqueued} base model(s) for registration.") diff --git a/backend/modelregistry/migrations/0004_localmodel_localmodel_status_valid_and_more.py b/backend/modelregistry/migrations/0004_localmodel_localmodel_status_valid_and_more.py new file mode 100644 index 000000000..c654a1954 --- /dev/null +++ b/backend/modelregistry/migrations/0004_localmodel_localmodel_status_valid_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.4 on 2026-07-22 11:40 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('modelregistry', '0003_localmodel_visibility_alter_localmodel_status'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddConstraint( + model_name='localmodel', + constraint=models.CheckConstraint(condition=models.Q(('status__in', ['active', 'archived'])), name='localmodel_status_valid'), + ), + migrations.AddConstraint( + model_name='localmodel', + constraint=models.CheckConstraint(condition=models.Q(('visibility__in', ['private', 'public'])), name='localmodel_visibility_valid'), + ), + ] diff --git a/backend/modelregistry/migrations/0005_basemodel.py b/backend/modelregistry/migrations/0005_basemodel.py new file mode 100644 index 000000000..82668bdce --- /dev/null +++ b/backend/modelregistry/migrations/0005_basemodel.py @@ -0,0 +1,36 @@ +# Generated by Django 6.0.4 on 2026-07-28 17:26 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('modelregistry', '0004_localmodel_localmodel_status_valid_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='BaseModel', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200, unique=True)), + ('category', models.CharField(choices=[('buildings', 'Buildings'), ('solar-panels', 'Solar Panels'), ('trees', 'Trees'), ('roads', 'Roads'), ('other', 'Other')], db_index=True, default='other', max_length=50)), + ('status', models.CharField(choices=[('registering', 'Registering'), ('active', 'Active'), ('failed', 'Failed'), ('archived', 'Archived')], default='registering', max_length=20)), + ('visibility', models.CharField(choices=[('private', 'Private'), ('public', 'Public')], db_index=True, default='public', max_length=20)), + ('stac_item', models.JSONField(blank=True, default=dict)), + ('error', models.TextField(blank=True, default='')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('last_modified', models.DateTimeField(auto_now=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='base_models', to=settings.AUTH_USER_MODEL, to_field='osm_id')), + ], + options={ + 'ordering': ['-created_at'], + 'indexes': [models.Index(fields=['status'], name='modelregist_status_f2a598_idx'), models.Index(fields=['user'], name='modelregist_user_id_b604eb_idx')], + 'constraints': [models.CheckConstraint(condition=models.Q(('status__in', ['registering', 'active', 'failed', 'archived'])), name='basemodel_status_valid'), models.CheckConstraint(condition=models.Q(('visibility__in', ['private', 'public'])), name='basemodel_visibility_valid')], + }, + ), + ] diff --git a/backend/modelregistry/migrations/0006_basemodel_stac_item_url_localmodel_category.py b/backend/modelregistry/migrations/0006_basemodel_stac_item_url_localmodel_category.py new file mode 100644 index 000000000..c7a646bf1 --- /dev/null +++ b/backend/modelregistry/migrations/0006_basemodel_stac_item_url_localmodel_category.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.4 on 2026-07-28 19:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('modelregistry', '0005_basemodel'), + ] + + operations = [ + migrations.AddField( + model_name='basemodel', + name='stac_item_url', + field=models.URLField(blank=True, default=''), + ), + migrations.AddField( + model_name='localmodel', + name='category', + field=models.CharField(choices=[('buildings', 'Buildings'), ('solar-panels', 'Solar Panels'), ('trees', 'Trees'), ('roads', 'Roads'), ('other', 'Other')], db_index=True, default='other', max_length=50), + ), + ] diff --git a/backend/modelregistry/models.py b/backend/modelregistry/models.py index fe73d9146..7e5491096 100644 --- a/backend/modelregistry/models.py +++ b/backend/modelregistry/models.py @@ -1,7 +1,12 @@ from django.db import models from accounts.models import OsmUser -from shared.enums import Visibility +from shared.enums import ( + BaseModelStatus, + LocalModelStatus, + ModelCategory, + Visibility, +) class LocalModel(models.Model): @@ -10,16 +15,20 @@ class LocalModel(models.Model): # creates a new STAC item under the same `mlm:name`. Per-version metadata # (title, description, assets) lives in STAC. - class Status(models.TextChoices): - ACTIVE = "active", "Active" - # TODO(archive-cascade): no endpoint flips a model to ARCHIVED yet. - # When added, must (1) archive_model_version per STAC item with - # mlm:name == self.name, (2) deprecate those STAC items, (3) mark - # related TrainingRunRefs (add `archived_at` field if needed). - ARCHIVED = "archived", "Archived" + # Module-level enum: a nested class body cannot see it from inside Meta. + Status = LocalModelStatus + Category = ModelCategory name = models.CharField(max_length=200, unique=True) - status = models.CharField(max_length=20, choices=Status.choices, default=Status.ACTIVE) + category = models.CharField( + max_length=50, + choices=ModelCategory.choices, + default=ModelCategory.OTHER, + db_index=True, + ) + status = models.CharField( + max_length=20, choices=LocalModelStatus.choices, default=LocalModelStatus.ACTIVE + ) visibility = models.CharField( max_length=20, choices=Visibility.choices, default=Visibility.PRIVATE, db_index=True ) @@ -37,6 +46,68 @@ class Meta: models.Index(fields=["status"]), models.Index(fields=["user"]), ] + constraints = [ + models.CheckConstraint( + condition=models.Q(status__in=LocalModelStatus.values), + name="localmodel_status_valid", + ), + models.CheckConstraint( + condition=models.Q(visibility__in=Visibility.values), + name="localmodel_visibility_valid", + ), + ] + ordering = ["-created_at"] + + def __str__(self) -> str: + return self.name + + +class BaseModel(models.Model): + Status = BaseModelStatus + Category = ModelCategory + + name = models.CharField(max_length=200, unique=True) + category = models.CharField( + max_length=50, + choices=ModelCategory.choices, + default=ModelCategory.OTHER, + db_index=True, + ) + status = models.CharField( + max_length=20, + choices=BaseModelStatus.choices, + default=BaseModelStatus.REGISTERING, + ) + visibility = models.CharField( + max_length=20, choices=Visibility.choices, default=Visibility.PUBLIC, db_index=True + ) + stac_item = models.JSONField(default=dict, blank=True) + stac_item_url = models.URLField(blank=True, default="") + error = models.TextField(blank=True, default="") + user = models.ForeignKey( + OsmUser, + to_field="osm_id", + on_delete=models.CASCADE, + related_name="base_models", + ) + created_at = models.DateTimeField(auto_now_add=True) + last_modified = models.DateTimeField(auto_now=True) + + class Meta: + indexes = [ + models.Index(fields=["status"]), + models.Index(fields=["user"]), + ] + constraints = [ + models.CheckConstraint( + condition=models.Q(status__in=BaseModelStatus.values), + name="basemodel_status_valid", + ), + models.CheckConstraint( + condition=models.Q(visibility__in=Visibility.values), + name="basemodel_visibility_valid", + ), + ] ordering = ["-created_at"] def __str__(self) -> str: diff --git a/backend/modelregistry/serializers.py b/backend/modelregistry/serializers.py index 096e154c7..8cb97a33a 100644 --- a/backend/modelregistry/serializers.py +++ b/backend/modelregistry/serializers.py @@ -1,8 +1,11 @@ +from typing import Any, cast + from rest_framework import serializers from notifications.serializers import UserSerializer +from shared.enums import ModelCategory -from .models import LocalModel +from .models import BaseModel, LocalModel class LocalModelSerializer(serializers.ModelSerializer): @@ -16,6 +19,7 @@ class Meta: fields = [ "id", "name", + "category", "status", "visibility", "user", @@ -36,6 +40,67 @@ class Meta: ] +class BaseModelSerializer(serializers.ModelSerializer): + user = UserSerializer(read_only=True) + + class Meta: + model = BaseModel + fields = [ + "id", + "name", + "category", + "status", + "visibility", + "stac_item_url", + "error", + "user", + "created_at", + "last_modified", + ] + read_only_fields = fields + + +class BaseModelRegisterSerializer(serializers.Serializer): + """Register a base model from an inline STAC item or a URL to one. + + Exactly one of ``stac_item`` (inline JSON) or ``stac_item_url`` (a link the + catalog can fetch) must be supplied; the row stores whichever was given. + """ + + stac_item = serializers.JSONField(required=False) + stac_item_url = serializers.URLField(required=False) + category = serializers.ChoiceField( + choices=ModelCategory.choices, + default=ModelCategory.OTHER, + ) + + def validate_stac_item(self, value: object) -> dict[str, Any]: + if not isinstance(value, dict): + raise serializers.ValidationError("stac_item must be a JSON object.") + item = cast("dict[str, Any]", value) + properties = item.get("properties") + name = ( + cast("dict[str, Any]", properties).get("mlm:name") + if isinstance(properties, dict) + else None + ) + if not name: + raise serializers.ValidationError("stac_item.properties['mlm:name'] is required.") + return item + + def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: + if bool(attrs.get("stac_item")) == bool(attrs.get("stac_item_url")): + raise serializers.ValidationError( + "Provide exactly one of 'stac_item' or 'stac_item_url'." + ) + return attrs + + +class BaseModelCategorySerializer(serializers.Serializer): + value = serializers.CharField() + label = serializers.CharField() + + class TrainingRunSummarySerializer(serializers.Serializer): """ZenML run summary, populated from fair.zenml.runs.RunSummary.""" diff --git a/backend/modelregistry/tasks.py b/backend/modelregistry/tasks.py new file mode 100644 index 000000000..cbc5333f0 --- /dev/null +++ b/backend/modelregistry/tasks.py @@ -0,0 +1,50 @@ +import json +import logging +import tempfile +from pathlib import Path + +from django_tasks import task + +from shared.integrations.stac import BASE_MODELS_COLLECTION, invalidate_stac_cache +from shared.integrations.zenml import for_user + +from .models import BaseModel + +logger = logging.getLogger(__name__) + + +@task() +def register_base_model(*, base_model_id: int) -> None: + """Register the row's stored STAC item via fair-py-ops and flip its status. + + Runs off-request because registration mirrors the model weights from the + source URLs into the artifact store, which can take minutes. + """ + # TODO: make fair-py-ops verify the inference URL from the STAC item instead + # of ensure_knative_service creating cluster resources on the prod path. + base_model = BaseModel.objects.get(id=base_model_id) + item_path: str | None = None + try: + if base_model.stac_item_url: + source = base_model.stac_item_url + elif base_model.stac_item: + handle, item_path = tempfile.mkstemp(suffix=".json") + with open(handle, "w") as fh: + json.dump(base_model.stac_item, fh) + source = item_path + else: + raise ValueError("base model has neither stac_item nor stac_item_url to register") + for_user(str(base_model.user.osm_id)).register_base_model(source) + base_model.status = BaseModel.Status.ACTIVE + base_model.error = "" + base_model.save(update_fields=["status", "error", "last_modified"]) + invalidate_stac_cache(BASE_MODELS_COLLECTION, base_model.name) + except Exception as exc: + logger.exception("base model registration failed for %s", base_model_id) + base_model.status = BaseModel.Status.FAILED + base_model.error = str(exc)[:2000] + base_model.save(update_fields=["status", "error", "last_modified"]) + raise + finally: + if item_path: + Path(item_path).unlink(missing_ok=True) diff --git a/backend/modelregistry/urls.py b/backend/modelregistry/urls.py index 92056f4c9..6cdd396e3 100644 --- a/backend/modelregistry/urls.py +++ b/backend/modelregistry/urls.py @@ -1,10 +1,11 @@ from django.urls import include, path from rest_framework import routers -from .views import LocalModelViewSet +from .views import BaseModelViewSet, LocalModelViewSet router = routers.DefaultRouter() router.register(r"local-models", LocalModelViewSet) +router.register(r"base-models", BaseModelViewSet) urlpatterns = [ path("", include(router.urls)), diff --git a/backend/modelregistry/views.py b/backend/modelregistry/views.py index 4230f0bbd..cea3c96d1 100644 --- a/backend/modelregistry/views.py +++ b/backend/modelregistry/views.py @@ -1,9 +1,11 @@ +import httpx from django.db.models import Count, Q from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import extend_schema, extend_schema_view -from rest_framework import filters, status, viewsets +from drf_spectacular.utils import OpenApiExample, extend_schema, extend_schema_view +from rest_framework import filters, mixins, status, viewsets from rest_framework.decorators import action -from rest_framework.permissions import IsAuthenticated +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.response import Response from accounts.authentication import OsmAuthentication @@ -13,7 +15,7 @@ PublishedReadOrAuthenticatedWrite, _is_admin, ) -from shared.enums import Visibility +from shared.enums import ModelCategory, Visibility from shared.integrations.stac import ( FAIR_PINNED_PROPERTY, LOCAL_MODELS_COLLECTION, @@ -23,8 +25,39 @@ from shared.integrations.zenml import list_runs_for_model from shared.stars import annotate_stars -from .models import LocalModel -from .serializers import LocalModelSerializer, TrainingRunSummarySerializer +from .models import BaseModel, LocalModel +from .serializers import ( + BaseModelCategorySerializer, + BaseModelRegisterSerializer, + BaseModelSerializer, + LocalModelSerializer, + TrainingRunSummarySerializer, +) +from .tasks import register_base_model + +_STAC_FETCH_TIMEOUT_S = 30 + + +def _fetch_mlm_name(url: str) -> str: + """Fetch the STAC item at `url` and return its ``mlm:name`` chain key. + + Registration stores only the link, so the name (unique per family) is read + once here to key the row. A bad URL or a payload without the field is a + caller error surfaced as a 400. + """ + try: + response = httpx.get(url, timeout=_STAC_FETCH_TIMEOUT_S, follow_redirects=True) + response.raise_for_status() + item = response.json() + except (httpx.HTTPError, ValueError) as exc: + raise ValidationError({"stac_item_url": f"could not fetch STAC item: {exc}"}) from exc + properties = item.get("properties") if isinstance(item, dict) else None + name = properties.get("mlm:name") if isinstance(properties, dict) else None + if not name: + raise ValidationError( + {"stac_item_url": "fetched STAC item is missing properties['mlm:name']."} + ) + return name @extend_schema_view( @@ -44,7 +77,7 @@ class LocalModelViewSet(viewsets.ReadOnlyModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [PublishedReadOrAuthenticatedWrite] filter_backends = [DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter] - filterset_fields = ["status", "visibility", "user"] + filterset_fields = ["status", "visibility", "category", "user"] search_fields = ["name"] ordering_fields = ["created_at", "last_modified"] @@ -118,3 +151,108 @@ def runs(self, request, pk: int | None = None) -> Response: for s in summaries ] return Response(TrainingRunSummarySerializer(data, many=True).data) + + +@extend_schema_view( + list=extend_schema(description="List registered base models (family records)."), + retrieve=extend_schema(description="Retrieve one base model by id."), + create=extend_schema( + description=( + "Register a base model from an inline STAC item JSON or a URL to one " + "(admin only). Supply exactly one of `stac_item` or `stac_item_url`. " + "Validates and publishes the item to the base-models collection " + "off-request; poll `status`." + ), + request=BaseModelRegisterSerializer, + responses={202: BaseModelSerializer}, + examples=[ + OpenApiExample( + "Inline STAC item", + value={ + "stac_item": { + "type": "Feature", + "id": "ramp-buildings", + "properties": {"mlm:name": "ramp-buildings"}, + "assets": {}, + }, + "category": "buildings", + }, + request_only=True, + ), + OpenApiExample( + "Link to a STAC item", + value={ + "stac_item_url": "https://example.com/models/ramp-buildings.json", + "category": "buildings", + }, + request_only=True, + ), + ], + ), +) +class BaseModelViewSet( + mixins.CreateModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + queryset = BaseModel.objects.all() + serializer_class = BaseModelSerializer + authentication_classes = [OsmAuthentication] + permission_classes = [IsAuthenticated] + filter_backends = [DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter] + filterset_fields = ["status", "visibility", "category", "user"] + search_fields = ["name"] + ordering_fields = ["created_at", "last_modified"] + + def get_permissions(self): + if self.action == "create": + return [IsAuthenticated(), IsAdmin()] + if self.action == "categories": + return [AllowAny()] + return [IsAuthenticated()] + + def create(self, request, *args, **kwargs) -> Response: + serializer = BaseModelRegisterSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + data = serializer.validated_data + category = data["category"] + stac_item = data.get("stac_item") or {} + stac_item_url = data.get("stac_item_url") or "" + name = stac_item["properties"]["mlm:name"] if stac_item else _fetch_mlm_name(stac_item_url) + + base_model, created = BaseModel.objects.get_or_create( + name=name, + defaults={ + "user": request.user, + "category": category, + "stac_item": stac_item, + "stac_item_url": stac_item_url, + "status": BaseModel.Status.REGISTERING, + }, + ) + if not created: + base_model.category = category + base_model.stac_item = stac_item + base_model.stac_item_url = stac_item_url + base_model.status = BaseModel.Status.REGISTERING + base_model.error = "" + base_model.save( + update_fields=[ + "category", + "stac_item", + "stac_item_url", + "status", + "error", + "last_modified", + ] + ) + + register_base_model.enqueue(base_model_id=base_model.id) + return Response(BaseModelSerializer(base_model).data, status=status.HTTP_202_ACCEPTED) + + @extend_schema(responses=BaseModelCategorySerializer(many=True)) + @action(detail=False, methods=["get"], url_path="categories") + def categories(self, request) -> Response: + data = [{"value": value, "label": label} for value, label in ModelCategory.choices] + return Response(BaseModelCategorySerializer(data, many=True).data) diff --git a/backend/predictions/migrations/0006_alter_prediction_status_and_more.py b/backend/predictions/migrations/0006_alter_prediction_status_and_more.py new file mode 100644 index 000000000..75feb29ad --- /dev/null +++ b/backend/predictions/migrations/0006_alter_prediction_status_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 6.0.4 on 2026-07-22 11:40 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('predictions', '0005_remove_prediction_is_public_prediction_visibility'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AlterField( + model_name='prediction', + name='status', + field=models.CharField(choices=[('initializing', 'Initializing'), ('submitted', 'Submitted'), ('provisioning', 'Provisioning'), ('running', 'Running'), ('completed', 'Completed'), ('failed', 'Failed'), ('cached', 'Cached'), ('retrying', 'Retrying'), ('retried', 'Retried'), ('stopping', 'Stopping'), ('stopped', 'Stopped')], default='initializing', max_length=20), + ), + migrations.AddConstraint( + model_name='prediction', + constraint=models.CheckConstraint(condition=models.Q(('status__in', ['initializing', 'submitted', 'provisioning', 'running', 'completed', 'failed', 'cached', 'retrying', 'retried', 'stopping', 'stopped'])), name='prediction_status_valid'), + ), + migrations.AddConstraint( + model_name='prediction', + constraint=models.CheckConstraint(condition=models.Q(('visibility__in', ['private', 'public'])), name='prediction_visibility_valid'), + ), + ] diff --git a/backend/predictions/models.py b/backend/predictions/models.py index 87bb6478a..0139554f4 100644 --- a/backend/predictions/models.py +++ b/backend/predictions/models.py @@ -1,7 +1,7 @@ from django.db import models from accounts.models import OsmUser -from shared.enums import Visibility +from shared.enums import PipelineRunStatus, Visibility class Prediction(models.Model): @@ -16,7 +16,11 @@ class Prediction(models.Model): max_length=20, choices=Visibility.choices, default=Visibility.PRIVATE, db_index=True ) description = models.TextField(blank=True) - status = models.CharField(max_length=20, default="initializing") + status = models.CharField( + max_length=20, + choices=PipelineRunStatus.choices, + default=PipelineRunStatus.INITIALIZING, + ) results_ready = models.BooleanField(default=False) mapswipe_project_id = models.CharField(max_length=100, blank=True) user = models.ForeignKey( @@ -34,4 +38,14 @@ class Meta: models.Index(fields=["status"]), models.Index(fields=["user"]), ] + constraints = [ + models.CheckConstraint( + condition=models.Q(status__in=PipelineRunStatus.values), + name="prediction_status_valid", + ), + models.CheckConstraint( + condition=models.Q(visibility__in=Visibility.values), + name="prediction_visibility_valid", + ), + ] ordering = ["-submitted_at"] diff --git a/backend/predictions/urls.py b/backend/predictions/urls.py index 04d014bf9..4d300173d 100644 --- a/backend/predictions/urls.py +++ b/backend/predictions/urls.py @@ -1,10 +1,11 @@ from django.urls import include, path from rest_framework import routers -from .views import PredictionViewSet +from .views import PredictionViewSet, PublicPredictionViewSet router = routers.DefaultRouter() router.register(r"predictions", PredictionViewSet) +router.register(r"public-predictions", PublicPredictionViewSet, basename="public-prediction") urlpatterns = [ path("", include(router.urls)), diff --git a/backend/predictions/views.py b/backend/predictions/views.py index e716d1bd3..9b42a5055 100644 --- a/backend/predictions/views.py +++ b/backend/predictions/views.py @@ -2,11 +2,16 @@ from django.db.models import Q from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import extend_schema, extend_schema_view, inline_serializer +from drf_spectacular.utils import ( + OpenApiExample, + extend_schema, + extend_schema_view, + inline_serializer, +) from rest_framework import filters, serializers, status, viewsets from rest_framework.decorators import action from rest_framework.exceptions import NotFound, PermissionDenied -from rest_framework.permissions import IsAuthenticated +from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.response import Response from rest_framework.throttling import ScopedRateThrottle @@ -98,6 +103,25 @@ def get_permissions(self): return [IsAuthenticated(), IsOwnerOrAdmin()] return super().get_permissions() + @extend_schema( + request=PredictionSubmitSerializer, + examples=[ + OpenApiExample( + "Submit prediction", + value={ + "model_stac_id": "0311d82d-0f8e-4021-adc5-bb4d6b81a1d4", + "image_uri": ( + "https://tiles.openaerialmap.org/62d85d11d8499800053796c1/0/" + "62d85d11d8499800053796c2/{z}/{x}/{y}" + ), + "bbox": [85.51678, 27.63133, 85.52323, 27.63743], + "zoom": 19, + "params": {"confidence_threshold": 0.25}, + }, + request_only=True, + ) + ], + ) @action( detail=False, methods=["post"], @@ -276,3 +300,23 @@ def _presigned_result_urls(prediction: Prediction) -> dict[str, str]: "fgb": presigned_get_url(StoragePaths.prediction_fgb_key(prediction.id)), "pmtiles": presigned_get_url(StoragePaths.prediction_pmtiles_key(prediction.id)), } + + +@extend_schema_view( + list=extend_schema( + tags=["public-predictions"], + description="List published (public) predictions. No authentication required.", + ), + retrieve=extend_schema( + tags=["public-predictions"], + description="Retrieve one published (public) prediction. No authentication required.", + ), +) +class PublicPredictionViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Prediction.objects.filter(visibility=Visibility.PUBLIC) + serializer_class = PredictionSerializer + authentication_classes: list = [] + permission_classes = [AllowAny] + filter_backends = [DjangoFilterBackend, filters.OrderingFilter] + filterset_fields = ["local_model_stac_id"] + ordering_fields = ["submitted_at"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index dec463a89..467a8da4b 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "djangorestframework>=3.15", "djangorestframework-gis>=1.1", "drf-spectacular>=0.29", - "fair-py-ops[k8s]==0.3.0", + "fair-py-ops[k8s]==0.3.2", "geomltoolkits>=2.1.0", "geopandas>=1.0", "gpxpy>=1.6", @@ -35,6 +35,7 @@ dependencies = [ "pyogrio>=0.10", "python-ulid>=3.1", "shapely>=2.0", + "whitenoise>=6.6", ] [dependency-groups] diff --git a/backend/shared/enums.py b/backend/shared/enums.py index 75f133afd..3324b7aa8 100644 --- a/backend/shared/enums.py +++ b/backend/shared/enums.py @@ -6,3 +6,55 @@ class Visibility(models.TextChoices): PRIVATE = "private", "Private" PUBLIC = "public", "Public" + + +class PipelineRunStatus(models.TextChoices): + """Lifecycle of a ZenML-backed training or prediction run. + + Mirrors ``fair.zenml.runs.RunStatus`` plus ``SUBMITTED``, which this backend + sets between handing a pipeline to ZenML and the first status poll. + """ + + INITIALIZING = "initializing", "Initializing" + SUBMITTED = "submitted", "Submitted" + PROVISIONING = "provisioning", "Provisioning" + RUNNING = "running", "Running" + COMPLETED = "completed", "Completed" + FAILED = "failed", "Failed" + CACHED = "cached", "Cached" + RETRYING = "retrying", "Retrying" + RETRIED = "retried", "Retried" + STOPPING = "stopping", "Stopping" + STOPPED = "stopped", "Stopped" + + +class DatasetStatus(models.TextChoices): + DRAFT = "draft", "Draft" + BUILDING = "building", "Building" + BUILT = "built", "Built" + FAILED = "failed", "Failed" + + +class LocalModelStatus(models.TextChoices): + ACTIVE = "active", "Active" + ARCHIVED = "archived", "Archived" + + +class BaseModelStatus(models.TextChoices): + REGISTERING = "registering", "Registering" + ACTIVE = "active", "Active" + FAILED = "failed", "Failed" + ARCHIVED = "archived", "Archived" + + +class ModelCategory(models.TextChoices): + BUILDINGS = "buildings", "Buildings" + SOLAR_PANELS = "solar-panels", "Solar Panels" + TREES = "trees", "Trees" + ROADS = "roads", "Roads" + OTHER = "other", "Other" + + +class FeedbackAction(models.TextChoices): + ACCEPT = "accept", "Accept" + REJECT = "reject", "Reject" diff --git a/backend/system/urls.py b/backend/system/urls.py index 94e8bae9c..87449c739 100644 --- a/backend/system/urls.py +++ b/backend/system/urls.py @@ -1,7 +1,8 @@ from django.urls import path -from .views import health +from .views import KpiStatsView, health urlpatterns = [ path("health/", health, name="health"), + path("kpi/stats/", KpiStatsView.as_view(), name="kpi-stats"), ] diff --git a/backend/system/views.py b/backend/system/views.py index 0c18af69a..787dbb869 100644 --- a/backend/system/views.py +++ b/backend/system/views.py @@ -10,8 +10,14 @@ from drf_spectacular.utils import extend_schema from rest_framework import serializers from rest_framework.decorators import throttle_classes +from rest_framework.permissions import AllowAny from rest_framework.response import Response +from rest_framework.views import APIView +from accounts.models import OsmUser +from feedback.models import Feedback +from modelregistry.models import LocalModel +from shared.enums import FeedbackAction, LocalModelStatus from shared.integrations.stac import ( BASE_MODELS_COLLECTION, DATASETS_COLLECTION, @@ -136,3 +142,39 @@ async def health(request) -> Response: } return Response(payload) + + +class KpiStatsSerializer(serializers.Serializer): + total_models_published = serializers.IntegerField() + total_registered_users = serializers.IntegerField() + total_feedback_labels = serializers.IntegerField() + total_accepted_predictions = serializers.IntegerField() + + +class KpiStatsView(APIView): + authentication_classes: list = [] + permission_classes = [AllowAny] + throttle_classes: list = [] + + @extend_schema( + tags=["system"], + responses=KpiStatsSerializer, + auth=[], + description=( + "Public homepage counters: published models, registered users, " + "feedback labels, accepted predictions." + ), + ) + def get(self, request) -> Response: + return Response( + { + "total_models_published": LocalModel.objects.filter( + status=LocalModelStatus.ACTIVE + ).count(), # TODO: should be localmodels+basemodels + "total_registered_users": OsmUser.objects.count(), + "total_feedback_labels": Feedback.objects.count(), + "total_accepted_predictions": Feedback.objects.filter( + action=FeedbackAction.ACCEPT + ).count(), + } + ) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 33020390d..9780dd251 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -137,6 +137,17 @@ def test_openapi_schema_is_served(anon_client): assert b"openapi" in response.content +def test_kpi_stats_returns_counts(anon_client, db): + response = anon_client.get("/api/v1/kpi/stats/") + assert response.status_code == 200 + assert set(response.data) == { + "total_models_published", + "total_registered_users", + "total_feedback_labels", + "total_accepted_predictions", + } + + def test_swagger_ui_renders(anon_client): response = anon_client.get("/api/docs/") assert response.status_code == 200 diff --git a/backend/tests/test_base_model_endpoints.py b/backend/tests/test_base_model_endpoints.py new file mode 100644 index 000000000..7544c9410 --- /dev/null +++ b/backend/tests/test_base_model_endpoints.py @@ -0,0 +1,144 @@ +import pytest +from rest_framework.test import APIClient + +from accounts.models import OsmUser +from modelregistry.models import BaseModel, LocalModel + +VALID_ITEM = { + "type": "Feature", + "id": "test-basemodel", + "properties": {"mlm:name": "test-basemodel"}, + "assets": {}, +} + + +class _FakeResponse: + def __init__(self, payload: dict) -> None: + self._payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return self._payload + + +@pytest.fixture +def user(db) -> OsmUser: + return OsmUser.objects.create(osm_id=41, username="mapper") + + +@pytest.fixture +def admin(db) -> OsmUser: + return OsmUser.objects.create(osm_id=42, username="boss", is_staff=True) + + +def _client(u: OsmUser) -> APIClient: + api = APIClient() + api.force_authenticate(user=u) + return api + + +def test_register_requires_admin(user: OsmUser) -> None: + resp = _client(user).post("/api/v1/base-models/", {"stac_item": VALID_ITEM}, format="json") + assert resp.status_code == 403 + assert not BaseModel.objects.exists() + + +def test_register_creates_row_and_returns_202(admin: OsmUser) -> None: + resp = _client(admin).post("/api/v1/base-models/", {"stac_item": VALID_ITEM}, format="json") + assert resp.status_code == 202 + base_model = BaseModel.objects.get(name="test-basemodel") + assert base_model.status == BaseModel.Status.REGISTERING + assert base_model.user_id == admin.osm_id + assert base_model.category == BaseModel.Category.OTHER + assert base_model.stac_item == VALID_ITEM + + +def test_register_stores_category(admin: OsmUser) -> None: + resp = _client(admin).post( + "/api/v1/base-models/", + {"stac_item": VALID_ITEM, "category": "buildings"}, + format="json", + ) + assert resp.status_code == 202 + assert BaseModel.objects.get(name="test-basemodel").category == "buildings" + + +def test_register_rejects_missing_mlm_name(admin: OsmUser) -> None: + resp = _client(admin).post( + "/api/v1/base-models/", {"stac_item": {"properties": {}}}, format="json" + ) + assert resp.status_code == 400 + assert not BaseModel.objects.exists() + + +def test_register_from_url_stores_link(admin: OsmUser, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "modelregistry.views.httpx.get", + lambda *args, **kwargs: _FakeResponse({"properties": {"mlm:name": "url-model"}}), + ) + resp = _client(admin).post( + "/api/v1/base-models/", + {"stac_item_url": "https://example.com/item.json", "category": "roads"}, + format="json", + ) + assert resp.status_code == 202 + base_model = BaseModel.objects.get(name="url-model") + assert base_model.stac_item_url == "https://example.com/item.json" + assert base_model.stac_item == {} + assert base_model.category == "roads" + + +def test_register_rejects_both_sources(admin: OsmUser) -> None: + resp = _client(admin).post( + "/api/v1/base-models/", + {"stac_item": VALID_ITEM, "stac_item_url": "https://example.com/item.json"}, + format="json", + ) + assert resp.status_code == 400 + assert not BaseModel.objects.exists() + + +def test_register_rejects_neither_source(admin: OsmUser) -> None: + resp = _client(admin).post("/api/v1/base-models/", {"category": "buildings"}, format="json") + assert resp.status_code == 400 + assert not BaseModel.objects.exists() + + +def test_register_rejects_unreachable_url(admin: OsmUser, monkeypatch: pytest.MonkeyPatch) -> None: + import httpx + + def _raise(*args, **kwargs): + raise httpx.ConnectError("boom") + + monkeypatch.setattr("modelregistry.views.httpx.get", _raise) + resp = _client(admin).post( + "/api/v1/base-models/", + {"stac_item_url": "https://example.com/item.json"}, + format="json", + ) + assert resp.status_code == 400 + assert not BaseModel.objects.exists() + + +def test_list_visible_to_authenticated(user: OsmUser, admin: OsmUser) -> None: + BaseModel.objects.create(name="m1", user=admin) + resp = _client(user).get("/api/v1/base-models/") + assert resp.status_code == 200 + assert resp.data["count"] == 1 + + +def test_categories_endpoint_is_public(db) -> None: + resp = APIClient().get("/api/v1/base-models/categories/") + assert resp.status_code == 200 + values = {c["value"] for c in resp.data} + assert {"buildings", "solar-panels", "other"} <= values + + +def test_local_model_has_category(admin: OsmUser) -> None: + model = LocalModel.objects.create(name="lm1", user=admin) + assert model.category == LocalModel.Category.OTHER + resp = _client(admin).get(f"/api/v1/local-models/{model.id}/") + assert resp.status_code == 200 + assert resp.data["category"] == "other" diff --git a/backend/tests/test_public_predictions.py b/backend/tests/test_public_predictions.py new file mode 100644 index 000000000..e0c4714f8 --- /dev/null +++ b/backend/tests/test_public_predictions.py @@ -0,0 +1,46 @@ +import pytest +from rest_framework.test import APIClient + +from accounts.models import OsmUser +from predictions.models import Prediction +from shared.enums import Visibility + +_TMS = "https://tiles.example.com/{z}/{x}/{y}" +_BBOX = [85.51678, 27.63133, 85.52323, 27.63743] + + +@pytest.fixture +def owner(db) -> OsmUser: + return OsmUser.objects.create(osm_id=8, username="dave") + + +def _prediction(owner: OsmUser, visibility: str) -> Prediction: + return Prediction.objects.create( + local_model_stac_id="m-1", + image_uri=_TMS, + bbox=_BBOX, + zoom=19, + visibility=visibility, + user=owner, + ) + + +def test_public_list_shows_only_public(owner: OsmUser) -> None: + public = _prediction(owner, Visibility.PUBLIC) + _prediction(owner, Visibility.PRIVATE) + resp = APIClient().get("/api/v1/public-predictions/") + assert resp.status_code == 200 + assert [p["id"] for p in resp.data["results"]] == [public.id] + + +def test_public_retrieve_public_ok(owner: OsmUser) -> None: + public = _prediction(owner, Visibility.PUBLIC) + resp = APIClient().get(f"/api/v1/public-predictions/{public.id}/") + assert resp.status_code == 200 + assert resp.data["visibility"] == Visibility.PUBLIC + + +def test_public_retrieve_private_is_404(owner: OsmUser) -> None: + private = _prediction(owner, Visibility.PRIVATE) + resp = APIClient().get(f"/api/v1/public-predictions/{private.id}/") + assert resp.status_code == 404 diff --git a/backend/trainings/migrations/0005_alter_trainingrunref_status_and_more.py b/backend/trainings/migrations/0005_alter_trainingrunref_status_and_more.py new file mode 100644 index 000000000..0a55960fc --- /dev/null +++ b/backend/trainings/migrations/0005_alter_trainingrunref_status_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 6.0.4 on 2026-07-22 11:40 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('datasets', '0004_dataset_dataset_status_valid_and_more'), + ('modelregistry', '0004_localmodel_localmodel_status_valid_and_more'), + ('trainings', '0004_remove_trainingrunref_trainings_t_status__77d437_idx_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AlterField( + model_name='trainingrunref', + name='status', + field=models.CharField(choices=[('initializing', 'Initializing'), ('submitted', 'Submitted'), ('provisioning', 'Provisioning'), ('running', 'Running'), ('completed', 'Completed'), ('failed', 'Failed'), ('cached', 'Cached'), ('retrying', 'Retrying'), ('retried', 'Retried'), ('stopping', 'Stopping'), ('stopped', 'Stopped')], default='initializing', max_length=20), + ), + migrations.AddConstraint( + model_name='trainingrunref', + constraint=models.CheckConstraint(condition=models.Q(('status__in', ['initializing', 'submitted', 'provisioning', 'running', 'completed', 'failed', 'cached', 'retrying', 'retried', 'stopping', 'stopped'])), name='trainingrunref_status_valid'), + ), + ] diff --git a/backend/trainings/models.py b/backend/trainings/models.py index 0cd97e14a..02ed6992d 100644 --- a/backend/trainings/models.py +++ b/backend/trainings/models.py @@ -3,6 +3,7 @@ from accounts.models import OsmUser from datasets.models import Dataset from modelregistry.models import LocalModel +from shared.enums import PipelineRunStatus class TrainingRunRef(models.Model): @@ -25,7 +26,11 @@ class TrainingRunRef(models.Model): # keywords + dataset keywords + dataset fair:geometry_type) at publish. keywords = models.JSONField(default=list, blank=True) description = models.TextField(blank=True) - status = models.CharField(max_length=20, default="initializing") + status = models.CharField( + max_length=20, + choices=PipelineRunStatus.choices, + default=PipelineRunStatus.INITIALIZING, + ) user = models.ForeignKey( OsmUser, to_field="osm_id", @@ -42,4 +47,10 @@ class Meta: models.Index(fields=["user"]), models.Index(fields=["base_model_stac_id"]), ] + constraints = [ + models.CheckConstraint( + condition=models.Q(status__in=PipelineRunStatus.values), + name="trainingrunref_status_valid", + ), + ] ordering = ["-submitted_at"] diff --git a/backend/trainings/views.py b/backend/trainings/views.py index 86298f547..ad956ced1 100644 --- a/backend/trainings/views.py +++ b/backend/trainings/views.py @@ -1,6 +1,11 @@ from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import extend_schema, extend_schema_view, inline_serializer +from drf_spectacular.utils import ( + OpenApiExample, + extend_schema, + extend_schema_view, + inline_serializer, +) from rest_framework import filters, serializers, status, viewsets from rest_framework.decorators import action from rest_framework.exceptions import NotFound, PermissionDenied @@ -83,6 +88,20 @@ class TrainingViewSet(viewsets.ReadOnlyModelViewSet): ordering_fields = ["submitted_at", "last_polled_at"] throttle_scope = "training_submit" + @extend_schema( + request=TrainingSubmitSerializer, + examples=[ + OpenApiExample( + "Submit finetune", + value={ + "base_model_stac_id": "unet-segmentation", + "dataset_stac_id": "banepa-buildings-1712345678-abcdef", + "model_name": "banepa-unet", + }, + request_only=True, + ) + ], + ) @action( detail=False, methods=["post"], @@ -216,6 +235,16 @@ def run_cancel(self, request, run_id: str) -> Response: name="TrainingPublishResponse", fields={"local_model_stac_id": serializers.CharField()}, ), + examples=[ + OpenApiExample( + "Publish trained model", + value={ + "title": "Banepa buildings UNet", + "description": "Promoted from training run", + }, + request_only=True, + ) + ], ) @action(detail=True, methods=["post"], url_path="publish") def publish(self, request, pk: int | None = None) -> Response: diff --git a/backend/uv.lock b/backend/uv.lock index be1774865..21b855dc4 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1021,6 +1021,7 @@ dependencies = [ { name = "pyogrio" }, { name = "python-ulid" }, { name = "shapely" }, + { name = "whitenoise" }, ] [package.dev-dependencies] @@ -1052,7 +1053,7 @@ requires-dist = [ { name = "djangorestframework", specifier = ">=3.15" }, { name = "djangorestframework-gis", specifier = ">=1.1" }, { name = "drf-spectacular", specifier = ">=0.29" }, - { name = "fair-py-ops", extras = ["k8s"], specifier = "==0.3.0" }, + { name = "fair-py-ops", extras = ["k8s"], specifier = "==0.3.2" }, { name = "geomltoolkits", specifier = ">=2.1.0" }, { name = "geopandas", specifier = ">=1.0" }, { name = "gpxpy", specifier = ">=1.6" }, @@ -1067,6 +1068,7 @@ requires-dist = [ { name = "pyogrio", specifier = ">=0.10" }, { name = "python-ulid", specifier = ">=3.1" }, { name = "shapely", specifier = ">=2.0" }, + { name = "whitenoise", specifier = ">=6.6" }, ] [package.metadata.requires-dev] @@ -1085,23 +1087,26 @@ test = [ [[package]] name = "fair-py-ops" -version = "0.3.0" +version = "0.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, { name = "mlflow" }, { name = "onnxruntime" }, { name = "pystac", extra = ["validation"] }, + { name = "reverse-geocode" }, + { name = "typer" }, { name = "universal-pathlib" }, { name = "zenml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/12/3a/87863266b4145fe92e4f4be83297fbafacfb3519ac18a21d8286c8bfc878/fair_py_ops-0.3.0.tar.gz", hash = "sha256:7c5530f7e86dbe7c2535cdb624f8863fcd83fa2eccecf1651a0af918dce4ed40", size = 80342, upload-time = "2026-05-18T13:54:28.201Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/68/6601800199b89189234886765195402c402d82a0bba5335ffbc2de881753/fair_py_ops-0.3.2.tar.gz", hash = "sha256:a3dcfcfa2f3f33bd02d543d9b9e1b23f0cb5978dc165682cb90d57b549faf8d8", size = 3927510, upload-time = "2026-07-22T16:53:11.084Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/46/f822386b258387194521e7a4365d93d26920181c9506ff77ff41122e56ca/fair_py_ops-0.3.0-py3-none-any.whl", hash = "sha256:45d7c737ab330f1408df52efd4c6d59043d09b96fd7c173c2a8d75d78416bd8d", size = 99790, upload-time = "2026-05-18T13:54:26.559Z" }, + { url = "https://files.pythonhosted.org/packages/02/d0/f9983a7f62f0cdc0fb2f19770170f4757622513947fc4e39a7f5e1763b98/fair_py_ops-0.3.2-py3-none-any.whl", hash = "sha256:7741644a316f1d6c16a4cc6ac4b82b290b59b0cd1b2aa816259ea18d8a6ad2c5", size = 4117702, upload-time = "2026-07-22T16:53:08.954Z" }, ] [package.optional-dependencies] k8s = [ + { name = "kubernetes" }, { name = "opentelemetry-proto" }, { name = "pypgstac", extra = ["psycopg"] }, { name = "pystac-client" }, @@ -1644,11 +1649,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/cc/87033e41f13fd4bdc003248a6fe65e880c406581605de7d2234c15a0317b/hydraters-0.1.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:0249bcfbdfca7bd41103bb162c3f51af200d430ff3203f9dced35c62ec5ecc72", size = 511151, upload-time = "2025-10-27T19:31:44.772Z" }, { url = "https://files.pythonhosted.org/packages/ca/27/5c9bcebf946842d8289f1591407b3a1387541eb220cf27c9b32378037f03/hydraters-0.1.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3a96e9b0a3be6ec2064f8556406fc4f225237fcaed1c118837a0ac9be2e62692", size = 437101, upload-time = "2025-10-27T19:31:55.632Z" }, { url = "https://files.pythonhosted.org/packages/ce/53/558519cd9b2888ef2859bef96d7338930377f3f629ec0f3b4df242083ec7/hydraters-0.1.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e4add7c611aae45e17eebda71f5d5d918dbb5515f4cbf33a3e8635a915f9b94", size = 408715, upload-time = "2025-10-27T19:32:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/0b/8b/72d8280bc2adc937162283ddde61286438f87d993b047db575100c0b9a60/hydraters-0.1.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:4d58ce6e5daa32f031306d7e141bf966702b590fb51fdfc70e7a0b69fac4f5de", size = 215054, upload-time = "2026-05-27T15:31:32.497Z" }, { url = "https://files.pythonhosted.org/packages/60/18/f5df44b3883295c342f8ba8bb7478ce36bd8de0513bc5c9d2aafa517dd88/hydraters-0.1.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:015770cba7cee9899f15fafc9486b77293c65dfd091da167d529546c8f2a4eee", size = 212845, upload-time = "2025-10-27T19:31:24.147Z" }, + { url = "https://files.pythonhosted.org/packages/f8/22/d49ce1ec0e6bbd45f6978c6208a53a9f02c0f2a82a76f3dba47717e507b7/hydraters-0.1.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dac8cf51e7bbadbfd94b87a023ad2f70e65c502bc82a8229aadaae1d08319589", size = 240388, upload-time = "2026-05-27T15:31:15.351Z" }, + { url = "https://files.pythonhosted.org/packages/e5/b1/e663b2ead344e1b91c15acb9eea71f57cb891fb23110acf355d1598ce082/hydraters-0.1.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6319f37972bf3fb63741b3a1bf3fddcaf42496f0249bce8692fc8f5da0dba304", size = 245641, upload-time = "2026-05-27T15:31:19.103Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f4/055365c2730c4adf68b0e7d15645a64a4a0448fae56a8ed957b1da2be159/hydraters-0.1.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a95367bad9fd2d3805412186d6512a5f5c9bb84c83ee208e5771e56dd82a6fea", size = 355334, upload-time = "2026-05-27T15:31:22.113Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3c/8e3a87e6e6bd500188fd23acbc4f2cc1a7f5582ed47473de39b7c607e10c/hydraters-0.1.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9893aa55e25ef1dc6bf13cd0ad7286e8a40138f67d61aee3f101e88f1e036083", size = 261409, upload-time = "2026-05-27T15:31:25.391Z" }, { url = "https://files.pythonhosted.org/packages/4a/65/6f775dd489fb85d5c2dbb9f50f7570130753b6a66b0263e9bc5dc0e42246/hydraters-0.1.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab670257e094d42e7cee1886109ea5af8ccb40eb4a3e8fc574550cad864276c2", size = 246841, upload-time = "2025-10-27T19:31:15.101Z" }, { url = "https://files.pythonhosted.org/packages/73/c1/0f2adcb91f0066b2ddcb59922adb2423a6b54d99d8c9169fea3f246e3082/hydraters-0.1.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0ed6a9760a7c5dd9323b354d26fc93b7819257f072300897efd37b518babbb9b", size = 257948, upload-time = "2025-10-27T19:31:05.641Z" }, + { url = "https://files.pythonhosted.org/packages/b6/dd/af7af983f1b7e6b77b0d340bb984d585b0d86c85eee03ac21cc4ea031190/hydraters-0.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:87d6b59b19a659d05385a80b141e55d06250786f5211f4d05580511dcfb99f5f", size = 416697, upload-time = "2026-05-27T15:31:34.176Z" }, + { url = "https://files.pythonhosted.org/packages/c9/22/1287974d201262202ea82940f3523b2bc4d507db7c3a6f97f83b7fb7e3db/hydraters-0.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d27f9f37c6ab7f1fbd286f7cc136c292c285b0109a12a1a9acbe25fbdd444e6e", size = 520655, upload-time = "2026-05-27T15:31:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/af/21/9fce9f979452bcaf27153611616b510505d3dc7057db8d0a057395da74b2/hydraters-0.1.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:5122acd52d97282a974d6fbef64bf8c8823082b39a26f7102bfdcbad9dcdfa3f", size = 477859, upload-time = "2026-05-27T15:31:40.335Z" }, + { url = "https://files.pythonhosted.org/packages/08/77/4c5dd49906f4212a6878424d575056566fcb27f017b79295a7f2254c6d3b/hydraters-0.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5edfe693599efa0bea4501534a34af5bcc572b6f791641f366d934be4d8511ef", size = 446221, upload-time = "2026-05-27T15:31:43.304Z" }, { url = "https://files.pythonhosted.org/packages/9e/37/c894a1e3cc2c19ac92d401957d703a620f726d2bab32c376c05c5ce3c75a/hydraters-0.1.3-cp314-cp314-win32.whl", hash = "sha256:be4492c6ed1d96cfa2cbc76b8e63163b878c51165c53712033906fb4539fd63a", size = 104209, upload-time = "2025-10-27T19:32:20.289Z" }, { url = "https://files.pythonhosted.org/packages/08/78/0ab4d10a51f512fa607501b67b1db53389465457b65cd275891537aebb7c/hydraters-0.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:0475e847adad810228f544da7b5e3ae9ab5c49e91f6fe37d39e47ac5e847a426", size = 109215, upload-time = "2025-10-27T19:32:17.724Z" }, + { url = "https://files.pythonhosted.org/packages/66/36/73e47d9df99e748aa5c102d372dd980ac01acaaf2bb2caa236031eb37c3d/hydraters-0.1.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebfc1c62578cddd4e3a4885c9decdc8e1c4be939c912bbb3c3f076894dc2a077", size = 238978, upload-time = "2026-05-27T15:31:17.02Z" }, + { url = "https://files.pythonhosted.org/packages/80/89/f0d24892ccfcb7995b3822b0c7fd4b3c00c23eb432da233b944d3ff0a920/hydraters-0.1.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:81bae487b7d9e0d22ec9786121a157cf0bd2b548ac4133bf60c56fc79b71db37", size = 244014, upload-time = "2026-05-27T15:31:20.07Z" }, + { url = "https://files.pythonhosted.org/packages/16/13/9448010fbf3b04a3c36e8cac75c154635d9620b45301c0e8359cf5453bb5/hydraters-0.1.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29e08aad3563fb67f04c37f6d5c659cc19f08b1f1e78078714536df86a0db777", size = 357545, upload-time = "2026-05-27T15:31:23.3Z" }, + { url = "https://files.pythonhosted.org/packages/51/da/919eb7244d3c50a419c9d7144dc16c32381ff6bf81091c2f687d47b0a45f/hydraters-0.1.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1f4cc518aa0f4ba61a1a7e8d89f51634dd3eafc2bbbb97f42b720693b79c0f6", size = 259992, upload-time = "2026-05-27T15:31:26.484Z" }, + { url = "https://files.pythonhosted.org/packages/5f/0e/7e5e4698c92b01117c04b08c886d48854445211b719d6c9e48c2bf4aec52/hydraters-0.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e881f653a171c70ed399634f18df009ff577a5bf53fb0842076aeea6a1f93533", size = 414898, upload-time = "2026-05-27T15:31:35.259Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2b/6857e0a70c0d84078856f1c30aceece4ed435de270a260ca19f19d8e01b6/hydraters-0.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:9fc1959a5c4c0d99f86512c474386f6a730dff71f82568298119444338805c09", size = 518992, upload-time = "2026-05-27T15:31:38.321Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/1c29c6c14dd57a82cd32a4d60474a90d770ec5c5962d70c8ea0fbf4d03a1/hydraters-0.1.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e1b616ec2435ff954b74d71ff5a0fd57631bdbb7b92373c6bc2c52e58b5940a9", size = 477171, upload-time = "2026-05-27T15:31:41.425Z" }, + { url = "https://files.pythonhosted.org/packages/76/e7/1eb86026cb43d1d9b2298c0420c8ab1fac710bd277388305807dadd2f854/hydraters-0.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:df5e7ffb1182cc474fd3e5fd6b92b847994310d7e58a9e93e058599ee75d1130", size = 445497, upload-time = "2026-05-27T15:31:44.473Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/fd12aee9f1a74df3fee23fe3d7aeb6a23be3050dd5b190c8165485d1d4ba/hydraters-0.1.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b11ab1a68d33e2dfb9679578253807aec4db4b85f6db35d10e4f10be6f64fc3", size = 243674, upload-time = "2026-05-27T15:31:30.412Z" }, + { url = "https://files.pythonhosted.org/packages/13/ed/a89eef1bb516aec2f3c0623da684559d5ce1721382a3e8ff8ab0834f196e/hydraters-0.1.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3c35323ba8e1e15615aa06ca76ec96035ba4717d4f0b0e15628ed01c37e10c1a", size = 257357, upload-time = "2026-05-27T15:31:28.539Z" }, ] [[package]] @@ -3455,6 +3479,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/04/7f73d05b556da048923e31a0cc878f03be7c5425ed1f268082255c75d872/responses-0.26.0-py3-none-any.whl", hash = "sha256:03ec4409088cd5c66b71ecbbbd27fe2c58ddfad801c66203457b3e6a04868c37", size = 35099, upload-time = "2026-02-19T14:38:03.847Z" }, ] +[[package]] +name = "reverse-geocode" +version = "1.6.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/a6/be1e9403cd4b668ff2ec7fd532b534f8f4b31e62637ce7476fbb20ae1b37/reverse_geocode-1.6.6.tar.gz", hash = "sha256:14165815816cc639dd74eb660a74e464aeebcd1d0814de74a897d62071c99f2a", size = 3469126, upload-time = "2025-05-15T08:43:34.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/94/8df60b4efe9edc8934218bac4764024ac423d2a2ed082bd0eb48d9321fe6/reverse_geocode-1.6.6-py3-none-any.whl", hash = "sha256:f639fdb99cc88c08d2ffaf39dfb1eb5a0ecc2860c50288c4613eea30826d6a9f", size = 3466565, upload-time = "2025-05-15T08:43:29.518Z" }, +] + [[package]] name = "rich" version = "15.0.0" @@ -4349,6 +4386,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, ] +[[package]] +name = "whitenoise" +version = "6.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/2a/55b3f3a4ec326cd077c1c3defeee656b9298372a69229134d930151acd01/whitenoise-6.12.0.tar.gz", hash = "sha256:f723ebb76a112e98816ff80fcea0a6c9b8ecde835f8ddda25df7a30a3c2db6ad", size = 26841, upload-time = "2026-02-27T00:05:42.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/eb/d5583a11486211f3ebd4b385545ae787f32363d453c19fffd81106c9c138/whitenoise-6.12.0-py3-none-any.whl", hash = "sha256:fc5e8c572e33ebf24795b47b6a7da8da3c00cff2349f5b04c02f28d0cc5a3cc2", size = 20302, upload-time = "2026-02-27T00:05:40.086Z" }, +] + [[package]] name = "wrapt" version = "2.1.2" diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 4cd79f27c..06645ad92 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: fair description: AI Assisted Mapping Tool type: application -version: 0.1.1 +version: 0.2.0 appVersion: "2.2.19" diff --git a/chart/README.md b/chart/README.md index d1c2d4033..5c9eb38f3 100644 --- a/chart/README.md +++ b/chart/README.md @@ -1,10 +1,11 @@ # fAIr Helm Chart -Deploys the fAIr backend API and Django-Q async worker. +Deploys the fAIr backend API and Django-Q async worker, and (optionally) the +frontend SPA. -The frontend is deployed separately (S3 + CloudFront via GitHub Actions). PostgreSQL is expected to be provided externally (e.g. -[CloudNativePG](https://cloudnative-pg.io/) or a managed database service). +[CloudNativePG](https://cloudnative-pg.io/) or a managed database service), with +a bundled single-pod Postgres available for staging / PR-preview / local dev. ## Quick start @@ -28,7 +29,7 @@ ingress: hosts: - host: fair.example.com paths: - - path: /api + - path: / pathType: Prefix backend: @@ -46,3 +47,27 @@ backend: | `backend.djangoQ.enabled` | Run Django-Q sidecar for async tasks | `true` | | `backend.migrate.enabled` | Run migrations on install/upgrade | `true` | | `ingress.enabled` | Create Ingress resource | `false` | +| `frontend.mode` | `"bundleWithBackend"` or `"cloudfront"` | `"bundleWithBackend"` | + +## Frontend + +The SPA is delivered by the `fair/frontend` image (`frontend/Dockerfile.prod`), +which is an init container that copies the built assets into a shared volume and +exits - not a running web server. `frontend.mode` picks how they're served: + +- **`bundleWithBackend`** (default) - the backend pod serves the SPA at `/` via + Django + WhiteNoise (`SERVE_FRONTEND=true`, set by the chart); API stays on + `/api`. One pod, one Service, one ingress path - good for staging, PR-preview + and simple self-hosted deploys with no AWS. +- **`cloudfront`** - a post-install/upgrade Job syncs the SPA to S3 and + creates/updates a CloudFront distribution, authenticating via IRSA + (`frontend.cloudfront.roleArn`). Frontend and API live on separate domains + (e.g. `fair.example.com` and `api.fair.example.com`). See `values.yaml` for the + full `frontend.cloudfront.*` options. + +Runtime config is injected at container start (written to `config.js`), so the +same image runs in any environment without a rebuild. This follows the same setup +as [hotosm/drone-tm](https://github.com/hotosm/drone-tm). For CloudFront +deployments, set `frontend.runtimeEnv.VITE_BASE_API_URL` to the absolute backend +API URL. The default build-time value remains `/api/v1/` for same-origin +deployments. diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index bfc7cfcda..024d245b6 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -5,7 +5,16 @@ Components: {{- if .Values.backend.djangoQ.enabled }} - Django-Q: running as sidecar in backend pod {{- end }} +{{- if eq (.Values.frontend.mode | default "bundleWithBackend") "bundleWithBackend" }} + - Frontend: bundled - the API pod serves the SPA at `/` (Django + WhiteNoise) +{{- else }} + - Frontend: deployed to S3 + CloudFront by the {{ include "fair.fullname" . }}-cloudfront-deploy Job +{{- end }} +{{- if .Values.postgres.enabled }} + - PostgreSQL: bundled ({{ include "fair.postgresName" . }}) +{{- else }} - PostgreSQL: external ({{ .Values.externalDatabase.host }}) +{{- end }} {{- if .Values.ingress.enabled }} diff --git a/chart/templates/backend/configmap.yaml b/chart/templates/backend/configmap.yaml index 72b047b70..580124919 100644 --- a/chart/templates/backend/configmap.yaml +++ b/chart/templates/backend/configmap.yaml @@ -6,6 +6,10 @@ metadata: {{- include "fair.labels" . | nindent 4 }} app.kubernetes.io/component: backend data: + {{- if eq (.Values.frontend.mode | default "bundleWithBackend") "bundleWithBackend" }} + # Serve the SPA baked into the fair/api image via Django/WhiteNoise. + SERVE_FRONTEND: "true" + {{- end }} {{- if not .Values.postgres.enabled }} DATABASE_HOST: {{ .Values.externalDatabase.host | quote }} DATABASE_PORT: {{ .Values.externalDatabase.port | toString | quote }} diff --git a/chart/templates/backend/deployment.yaml b/chart/templates/backend/deployment.yaml index 8e202dde7..8704c9fd4 100644 --- a/chart/templates/backend/deployment.yaml +++ b/chart/templates/backend/deployment.yaml @@ -1,3 +1,4 @@ +{{- $bundle := eq (.Values.frontend.mode | default "bundleWithBackend") "bundleWithBackend" -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -29,6 +30,22 @@ spec: securityContext: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $bundle }} + initContainers: + # Copies the built SPA into the shared volume the API pod serves via + # Django/WhiteNoise (SERVE_FRONTEND=true, see configmap). + - name: frontend-assets + image: "{{ .Values.frontend.image.repository }}:{{ .Values.frontend.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.frontend.image.pullPolicy }} + env: + {{- range $key, $value := (.Values.frontend.runtimeEnv | default dict) }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + volumeMounts: + - name: frontend-html + mountPath: /frontend_html + {{- end }} containers: - name: api image: {{ include "fair.backend.image" . }} @@ -48,6 +65,10 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} env: + {{- range $key, $val := .Values.backend.env }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end }} {{- if .Values.postgres.enabled }} {{- include "fair.postgresDatabaseUrlEnv" . | nindent 12 }} {{- else }} @@ -64,6 +85,12 @@ spec: - name: DATABASE_URL value: {{ include "fair.databaseUrl" . | quote }} {{- end }} + {{- if $bundle }} + volumeMounts: + - name: frontend-html + mountPath: /app/frontend_html + readOnly: true + {{- end }} {{- with .Values.backend.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} @@ -87,7 +114,7 @@ spec: command: - python - manage.py - - qcluster + - db_worker envFrom: - configMapRef: name: {{ include "fair.backend.fullname" . }} @@ -120,6 +147,11 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} + {{- if $bundle }} + volumes: + - name: frontend-html + emptyDir: {} + {{- end }} {{- with .Values.backend.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/chart/templates/frontend/cloudfront-deploy-job.yaml b/chart/templates/frontend/cloudfront-deploy-job.yaml new file mode 100644 index 000000000..06b911c37 --- /dev/null +++ b/chart/templates/frontend/cloudfront-deploy-job.yaml @@ -0,0 +1,208 @@ +{{- if eq (.Values.frontend.mode | default "bundleWithBackend") "cloudfront" }} +{{- $cf := .Values.frontend.cloudfront }} +{{- $version := $cf.version | default .Chart.AppVersion }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "fair.fullname" . }}-cloudfront-deploy + labels: + {{- include "fair.labels" . | nindent 4 }} + app.kubernetes.io/component: cloudfront-deploy + annotations: + # Run after the backend rollout (migrate hook is weight -5) so the API is + # live before the new frontend starts serving traffic. + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 3 + template: + metadata: + labels: + {{- include "fair.labels" . | nindent 8 }} + app.kubernetes.io/component: cloudfront-deploy + spec: + restartPolicy: Never + serviceAccountName: {{ include "fair.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + # Copies the built SPA into the shared volume for the aws-cli sync step. + - name: frontend-assets + image: "{{ .Values.frontend.image.repository }}:{{ .Values.frontend.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.frontend.image.pullPolicy }} + env: + {{- range $key, $value := (.Values.frontend.runtimeEnv | default dict) }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + volumeMounts: + - name: frontend-html + mountPath: /frontend_html + containers: + - name: deploy + image: "{{ $cf.awsCliImage.repository }}:{{ $cf.awsCliImage.tag }}" + imagePullPolicy: {{ $cf.awsCliImage.pullPolicy }} + env: + - name: AWS_DEFAULT_REGION + value: {{ $cf.region | quote }} + command: + - /bin/bash + - -euo + - pipefail + - -c + - | + set -x + + BUCKET={{ $cf.s3Bucket | quote }} + VERSION={{ $version | quote }} + RELEASE={{ include "fair.fullname" . | quote }} + + # ── 1. Sync frontend to versioned S3 path ────────────────────── + echo "Syncing to s3://${BUCKET}/${VERSION}/" + aws s3 sync /frontend_html/ "s3://${BUCKET}/${VERSION}/" \ + --delete \ + --cache-control "public, max-age=31536000, immutable" + + # index.html and config.js must not be cached long (mutable on each deploy). + declare -A CONTENT_TYPES=( [index.html]="text/html" [config.js]="application/javascript" ) + for f in index.html config.js; do + if aws s3 ls "s3://${BUCKET}/${VERSION}/${f}" 2>/dev/null; then + aws s3 cp "s3://${BUCKET}/${VERSION}/${f}" "s3://${BUCKET}/${VERSION}/${f}" \ + --cache-control "public, max-age=60" \ + --content-type "${CONTENT_TYPES[$f]}" \ + --metadata-directive REPLACE + fi + done + + # ── 2. Find or create CloudFront distribution ─────────────────── + echo "Looking up CloudFront distribution for ${BUCKET}..." + DIST_ID=$(aws cloudfront list-distributions \ + --query "DistributionList.Items[?Origins.Items[0].DomainName=='${BUCKET}.s3.amazonaws.com'].Id | [0]" \ + --output text 2>/dev/null || true) + [ "${DIST_ID}" = "None" ] && DIST_ID="" + + if [ -n "${DIST_ID}" ]; then + echo "Found existing distribution ${DIST_ID}" + else + echo "No distribution found - creating one..." + + # Origin Access Control (find or create by name). + OAC_NAME="${RELEASE}-oac" + OAC_ID=$(aws cloudfront list-origin-access-controls \ + --query "OriginAccessControlList.Items[?Name=='${OAC_NAME}'].Id | [0]" \ + --output text 2>/dev/null || true) + [ "${OAC_ID}" = "None" ] && OAC_ID="" + + if [ -z "${OAC_ID}" ]; then + OAC_ID=$(aws cloudfront create-origin-access-control \ + --origin-access-control-config \ + "Name=${OAC_NAME},SigningProtocol=sigv4,SigningBehavior=always,OriginAccessControlOriginType=s3,Description=Managed by fair helm chart" \ + --query "OriginAccessControl.Id" --output text) + echo "Created OAC ${OAC_ID}" + fi + + # Build distribution config via Python (cleaner than inline JSON escaping). + DIST_CONFIG=$(python3 -c " + import json, time + config = { + 'CallerReference': f'fair-{int(time.time())}', + 'Comment': '${RELEASE} frontend', + 'Enabled': True, + 'DefaultRootObject': 'index.html', + 'Origins': { + 'Quantity': 1, + 'Items': [{ + 'Id': 's3-${BUCKET}', + 'DomainName': '${BUCKET}.s3.amazonaws.com', + 'OriginPath': '/${VERSION}', + 'S3OriginConfig': {'OriginAccessIdentity': ''}, + 'OriginAccessControlId': '${OAC_ID}', + }], + }, + 'DefaultCacheBehavior': { + 'TargetOriginId': 's3-${BUCKET}', + 'ViewerProtocolPolicy': 'redirect-to-https', + 'CachePolicyId': '658327ea-f89d-4fab-a63d-7e88639e58f6', + 'Compress': True, + }, + 'CustomErrorResponses': { + 'Quantity': 2, + 'Items': [ + {'ErrorCode': 403, 'ResponseCode': '200', 'ResponsePagePath': '/index.html', 'ErrorCachingMinTTL': 60}, + {'ErrorCode': 404, 'ResponseCode': '200', 'ResponsePagePath': '/index.html', 'ErrorCachingMinTTL': 60}, + ], + }, + 'PriceClass': '{{ $cf.priceClass }}', + } + {{- $aliases := $cf.aliases | default list }} + {{- if $aliases }} + config['Aliases'] = {'Quantity': {{ len $aliases }}, 'Items': [{{ range $i, $a := $aliases }}{{ if $i }}, {{ end }}'{{ $a }}'{{ end }}]} + config['ViewerCertificate'] = { + 'ACMCertificateArn': '{{ $cf.acmCertificateArn }}', + 'SSLSupportMethod': 'sni-only', + 'MinimumProtocolVersion': 'TLSv1.2_2021', + } + {{- else }} + config['Aliases'] = {'Quantity': 0, 'Items': []} + config['ViewerCertificate'] = {'CloudFrontDefaultCertificate': True} + {{- end }} + print(json.dumps(config)) + ") + + DIST_ARN=$(aws cloudfront create-distribution \ + --distribution-config "${DIST_CONFIG}" \ + --query "Distribution.ARN" --output text) + DIST_ID=$(echo "${DIST_ARN}" | awk -F/ '{print $NF}') + echo "Created distribution ${DIST_ID}" + + # Grant CloudFront OAC read access to the S3 bucket. + BUCKET_POLICY=$(python3 -c " + import json + print(json.dumps({ + 'Version': '2012-10-17', + 'Statement': [{ + 'Sid': 'AllowCloudFrontOAC', + 'Effect': 'Allow', + 'Principal': {'Service': 'cloudfront.amazonaws.com'}, + 'Action': 's3:GetObject', + 'Resource': 'arn:aws:s3:::${BUCKET}/*', + 'Condition': { + 'StringEquals': { + 'AWS:SourceArn': '${DIST_ARN}' + } + } + }] + })) + ") + aws s3api put-bucket-policy --bucket "${BUCKET}" --policy "${BUCKET_POLICY}" + echo "S3 bucket policy updated for OAC access" + fi + + # ── 3. Update origin path to the new version ──────────────────── + echo "Updating origin path to /${VERSION}..." + ETAG=$(aws cloudfront get-distribution-config --id "${DIST_ID}" --query "ETag" --output text) + UPDATED_CONFIG=$(aws cloudfront get-distribution-config --id "${DIST_ID}" --query "DistributionConfig" \ + | python3 -c " + import json, sys + config = json.load(sys.stdin) + config['Origins']['Items'][0]['OriginPath'] = '/${VERSION}' + print(json.dumps(config)) + ") + aws cloudfront update-distribution --id "${DIST_ID}" --if-match "${ETAG}" --distribution-config "${UPDATED_CONFIG}" + + # ── 4. Invalidate CloudFront cache ────────────────────────────── + echo "Invalidating cache..." + aws cloudfront create-invalidation --distribution-id "${DIST_ID}" --paths "/*" + + echo "Done - version=${VERSION} distribution=${DIST_ID}" + volumeMounts: + - name: frontend-html + mountPath: /frontend_html + readOnly: true + volumes: + - name: frontend-html + emptyDir: {} +{{- end }} diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml index 275a251e6..41a3ae6d9 100644 --- a/chart/templates/serviceaccount.yaml +++ b/chart/templates/serviceaccount.yaml @@ -5,8 +5,15 @@ metadata: name: {{ include "fair.serviceAccountName" . }} labels: {{- include "fair.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} + {{- $irsa := and (eq (.Values.frontend.mode | default "bundleWithBackend") "cloudfront") .Values.frontend.cloudfront.roleArn }} + {{- if or .Values.serviceAccount.annotations $irsa }} annotations: + {{- if $irsa }} + # IRSA: lets the CloudFront deploy Job assume the IAM role without static keys. + eks.amazonaws.com/role-arn: {{ .Values.frontend.cloudfront.roleArn | quote }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index 0416ebea1..0e4f77a39 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -3,7 +3,7 @@ backendReplicaCount: 1 image: backend: - repository: ghcr.io/hotosm/fair-api + repository: ghcr.io/hotosm/fair/api tag: "" # defaults to appVersion pullPolicy: IfNotPresent @@ -23,7 +23,7 @@ backend: - --bind=0.0.0.0:8000 - --workers=3 - --timeout=120 - - fairproject.wsgi:application + - config.wsgi:application # -- Run Django-Q cluster as a sidecar for lightweight async tasks djangoQ: @@ -60,16 +60,22 @@ backend: livenessProbe: httpGet: - path: /api/ + path: /api port: http + httpHeaders: + - name: Host + value: localhost initialDelaySeconds: 30 periodSeconds: 30 timeoutSeconds: 5 readinessProbe: httpGet: - path: /api/ + path: /api port: http + httpHeaders: # use localhost rather than hosts IP + - name: Host + value: localhost initialDelaySeconds: 15 periodSeconds: 10 timeoutSeconds: 5 @@ -89,7 +95,44 @@ backend: type: ClusterIP port: 8000 -# -- Ingress configuration +# -- Frontend deployment. The SPA comes from the fair/frontend image, which runs +# as an init container (copies assets into a shared volume and exits; not a web +# server). See frontend/Dockerfile.prod. Two mutually exclusive modes: +# "bundleWithBackend" - Django/WhiteNoise serves the SPA at `/`, same origin as +# the API. Sets SERVE_FRONTEND=true automatically. Default. +# "cloudfront" - a Helm Job syncs the SPA to S3 + CloudFront, using IRSA +# (no static AWS keys). +# Deployment-specific config can be set via `frontend.runtimeEnv`. +frontend: + image: + repository: ghcr.io/hotosm/fair/frontend + tag: "" # defaults to appVersion + pullPolicy: IfNotPresent + + # Env written into SPA at container start (config.js), no img rebuild needed + runtimeEnv: {} + # VITE_BASE_API_URL: "https://api.example.com/api/v1/" + # VITE_AUTH_PROVIDER: "hanko" + # VITE_HANKO_URL: "https://login.hotosm.org" + + mode: "bundleWithBackend" + + # Settings for mode: cloudfront + cloudfront: + roleArn: "" # IAM role ARN for IRSA (annotated onto the ServiceAccount) + region: "us-east-1" + s3Bucket: "" # Required, e.g. "fair-frontend" + version: "" # S3 path prefix; defaults to appVersion. Set to an older value to rollback. + aliases: [] # e.g. ["fair.hotosm.org"]; only used when creating a new distribution + acmCertificateArn: "" # Required when aliases are set + priceClass: "PriceClass_All" + awsCliImage: + repository: docker.io/amazon/aws-cli + tag: "2.34.17" + pullPolicy: IfNotPresent + +# -- Ingress configuration. In bundleWithBackend mode the backend serves both the +# SPA (`/`) and API (`/api`). In cloudfront mode point the ingress at `/api`. ingress: enabled: false className: "" @@ -98,7 +141,7 @@ ingress: hosts: - host: fair.example.com paths: - - path: /api + - path: / pathType: Prefix tls: [] # - secretName: fair-tls diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 000000000..a69fb6d35 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,77 @@ +# fair-dev instance override, layered on top of docker-compose.yml. +services: + frontend-assets: + environment: + VITE_BASE_API_URL: ${VITE_BASE_API_URL} + VITE_AUTH_PROVIDER: ${VITE_AUTH_PROVIDER} + VITE_HANKO_URL: ${VITE_HANKO_URL} + VITE_FAIR_STAC_CATALOG_BASE_URL: ${VITE_FAIR_STAC_CATALOG_BASE_URL} + VITE_NODE_ENV: ${VITE_NODE_ENV} + VITE_FAIR_PROD_URL: ${VITE_FAIR_PROD_URL} + + caddy: + image: caddy:2-alpine + restart: unless-stopped + ports: + - "80:80" + - "443:443" + configs: + - source: caddyfile + target: /etc/caddy/Caddyfile + volumes: + - caddy_data:/data + - caddy_config:/config + depends_on: + - api + + postgres: + restart: unless-stopped + + minio: + restart: unless-stopped + + stac: + restart: unless-stopped + + mlflow: + restart: unless-stopped + + zenml: + restart: unless-stopped + + api: + restart: unless-stopped + entrypoint: ["sh", "-c", 'zenml init && zenml stack set fair-compose && python manage.py collectstatic --noinput && exec "$$@"', "--"] + command: + - gunicorn + - --bind=0.0.0.0:8000 + - --workers=3 + - --timeout=120 + - config.wsgi:application + + worker: + restart: unless-stopped + +configs: + caddyfile: + content: | + { + email ${CADDY_ACME_EMAIL:-sysadmin@hotosm.org} + } + ${PUBLIC_DOMAIN:-dev.ai.hotosm.org}, fair-dev.hotosm.org { + handle_path /stac/* { + reverse_proxy stac:8080 + } + reverse_proxy api:8000 + } + stac.${PUBLIC_DOMAIN:-dev.ai.hotosm.org} { + reverse_proxy stac:8080 + } + # mlflow.${PUBLIC_DOMAIN:-dev.ai.hotosm.org} { reverse_proxy mlflow:5000 } + # zenml.${PUBLIC_DOMAIN:-dev.ai.hotosm.org} { reverse_proxy zenml:8080 } + # minio.${PUBLIC_DOMAIN:-dev.ai.hotosm.org} { reverse_proxy minio:9000 } + # console.${PUBLIC_DOMAIN:-dev.ai.hotosm.org} { reverse_proxy minio:9001 } + +volumes: + caddy_data: + caddy_config: diff --git a/docker-compose.hotreload.yml b/docker-compose.hotreload.yml new file mode 100644 index 000000000..dd79b5d8c --- /dev/null +++ b/docker-compose.hotreload.yml @@ -0,0 +1,18 @@ +# Opt-in local hot reload: bind-mounts ./backend into the api and worker +# TODO : add frontend support +# Enable by appending this file to COMPOSE_FILE, e.g.: +# COMPOSE_FILE=docker-compose.yml:docker-compose.hotreload.yml docker compose up + + +services: + api: + volumes: + - ./backend:/app + - /app/.venv + - /app/.git + + worker: + volumes: + - ./backend:/app + - /app/.venv + - /app/.git diff --git a/docker-compose.yml b/docker-compose.yml index 76b4e769e..3ccd8fa94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,19 @@ +x-backend: &backend + image: ghcr.io/hotosm/fair/api:develop + build: + context: ./backend + env_file: + - ./.env + +# FairClient.setup() re-runs `zenml init`, which resets the active stack unless +# the repo is already initialised, so api and worker prime it before starting. +x-zenml-entrypoint: &zenml-entrypoint + entrypoint: ["sh", "-c", 'zenml init && zenml stack set fair-compose && exec "$$@"', "--"] + +networks: + default: + name: fair-net + services: postgres: image: postgis/postgis:17-3.5-alpine @@ -10,47 +26,266 @@ services: volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER:-admin}"] + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER:-admin}", "-d", "${POSTGRES_DB:-fair}"] interval: 5s timeout: 3s retries: 10 - api: - build: - context: ./backend - env_file: - - ./backend/.env - ports: - - "${API_PORT:-8000}:8000" + db-init: + image: postgis/postgis:17-3.5-alpine depends_on: postgres: condition: service_healthy + environment: + PGHOST: postgres + PGUSER: ${POSTGRES_USER:-admin} + PGPASSWORD: ${POSTGRES_PASSWORD:-password} + PGDATABASE: ${POSTGRES_DB:-fair} + command: + - sh + - -c + - | + set -e + for db in zenml mlflow stac; do + psql -tAc "SELECT 1 FROM pg_database WHERE datname='$$db'" | grep -q 1 || + psql -c "CREATE DATABASE $$db" + done + psql -d stac -c "CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist" + + minio: + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + ports: + - "${MINIO_PORT:-9000}:9000" + - "${MINIO_CONSOLE_PORT:-9001}:9001" volumes: - - ./backend:/app - command: ["python", "manage.py", "runserver", "0.0.0.0:8000"] + - minio_data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 3s + retries: 30 - worker: - build: - context: ./backend - env_file: - - ./backend/.env + minio-init: + image: minio/mc:RELEASE.2025-08-13T08-35-41Z depends_on: - postgres: + minio: condition: service_healthy - volumes: - - ./backend:/app - command: ["python", "manage.py", "db_worker"] + entrypoint: + - sh + - -c + - | + set -e + mc alias set --quiet fair http://minio:9000 ${MINIO_ROOT_USER:-minioadmin} ${MINIO_ROOT_PASSWORD:-minioadmin} + mc mb --ignore-existing fair/fair-data fair/mlflow fair/zenml + for b in fair-data zenml; do mc anonymous set download "fair/$$b"; done + + pgstac-migrate: + image: ghcr.io/stac-utils/pgstac-pypgstac:v0.9.9 + depends_on: + db-init: + condition: service_completed_successfully + environment: + PGHOST: postgres + PGPORT: "5432" + PGUSER: ${POSTGRES_USER:-admin} + PGPASSWORD: ${POSTGRES_PASSWORD:-password} + PGDATABASE: stac + command: ["pypgstac", "migrate"] + + stac: + image: ghcr.io/stac-utils/stac-fastapi-pgstac:5.0.2 + depends_on: + pgstac-migrate: + condition: service_completed_successfully + environment: + POSTGRES_HOST_READER: postgres + POSTGRES_HOST_WRITER: postgres + POSTGRES_PORT: "5432" + POSTGRES_USER: ${POSTGRES_USER:-admin} + POSTGRES_PASS: ${POSTGRES_PASSWORD:-password} + POSTGRES_DBNAME: stac + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + ENABLE_TRANSACTIONS_EXTENSIONS: "TRUE" + ports: + - "${STAC_PORT:-8082}:8080" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/_mgmt/ping')"] + interval: 5s + timeout: 3s + retries: 30 - frontend: - profiles: ["dev"] + mlflow: + image: burakince/mlflow:3.7.0 + depends_on: + db-init: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + environment: + MLFLOW_S3_ENDPOINT_URL: http://minio:9000 + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-minioadmin} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-minioadmin} + command: + - mlflow + - server + - --host=0.0.0.0 + - --port=5000 + - --backend-store-uri=postgresql://${POSTGRES_USER:-admin}:${POSTGRES_PASSWORD:-password}@postgres:5432/mlflow + - --default-artifact-root=s3://mlflow/ + - --serve-artifacts + - --allowed-hosts=mlflow:5000,localhost:5000 + ports: + - "${MLFLOW_PORT:-5000}:5000" + + zenml: + image: ghcr.io/hotosm/fair/zenml-postgres:0.94.2 + depends_on: + db-init: + condition: service_completed_successfully + environment: + ZENML_STORE_URL: postgresql://${POSTGRES_USER:-admin}:${POSTGRES_PASSWORD:-password}@postgres:5432/zenml + ZENML_SERVER_AUTO_ACTIVATE: "1" + ZENML_ANALYTICS_OPT_IN: "false" + ports: + - "${ZENML_PORT:-8080}:8080" + + zenml-init: + <<: *backend + depends_on: + zenml: + condition: service_started + minio-init: + condition: service_completed_successfully + configs: + - source: zenml_stack + target: /tmp/fair-compose.yaml + command: + - sh + - -c + - | + set -e + until zenml stack list; do sleep 5; done + zenml stack describe fair-compose || + zenml stack import fair-compose -f /tmp/fair-compose.yaml --ignore-version-mismatch + + # Registers the base models + stac-seed: + <<: *backend + depends_on: + stac: + condition: service_healthy + environment: + FAIR_USER_ID: fair-seed + command: + - sh + - -c + - | + set -e + root=$$(python -c "import models, pathlib; print(pathlib.Path(models.__file__).parent)") + registered=$$(fair item list base-models | cut -f1) + for item in "$$root"/*/stac-item.json; do + id=$$(python -c "import json, sys; print(json.load(open(sys.argv[1]))['id'])" "$$item") + if echo "$$registered" | grep -Fxq "$$id"; then + echo "seed: $$id already registered" + else + fair basemodel register "$$item" + fi + done + + frontend-assets: + image: ghcr.io/hotosm/fair/frontend:develop build: context: ./frontend - dockerfile: Dockerfile.dev + dockerfile: Dockerfile.prod + environment: + VITE_FAIR_STAC_CATALOG_BASE_URL: ${VITE_FAIR_STAC_CATALOG_BASE_URL:-http://localhost:${STAC_PORT:-8082}/} + volumes: + - frontend_html:/frontend_html + + migrate: + <<: *backend + depends_on: + postgres: + condition: service_healthy + command: ["python", "manage.py", "migrate", "--noinput"] + + api: + <<: [*backend, *zenml-entrypoint] + environment: + SERVE_FRONTEND: "true" + FRONTEND_DIST_DIR: /frontend_html ports: - - "${FRONTEND_PORT:-3500}:3000" + - "${API_PORT:-8000}:8000" + depends_on: + migrate: + condition: service_completed_successfully + frontend-assets: + condition: service_completed_successfully + zenml-init: + condition: service_completed_successfully + stac-seed: + condition: service_completed_successfully + volumes: + - frontend_html:/frontend_html + command: ["python", "manage.py", "runserver", "0.0.0.0:8000"] + + worker: + <<: [*backend, *zenml-entrypoint] + depends_on: + migrate: + condition: service_completed_successfully + zenml-init: + condition: service_completed_successfully volumes: - - ./frontend:/app - - /app/node_modules + - /var/run/docker.sock:/var/run/docker.sock + command: ["python", "manage.py", "db_worker"] + +configs: + zenml_stack: + content: | + stack_name: fair-compose + zenml_version: 0.94.2 + components: + orchestrator: + name: docker_compose + flavor: local_docker + type: orchestrator + configuration: + synchronous: false + run_args: + network: fair-net + user: "0" + environment: + AWS_ENDPOINT_URL: http://minio:9000 + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-minioadmin} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-minioadmin} + NVIDIA_VISIBLE_DEVICES: all + NVIDIA_DRIVER_CAPABILITIES: compute,utility + artifact_store: + name: s3_compose + flavor: s3 + type: artifact_store + configuration: + path: s3://zenml + key: ${MINIO_ROOT_USER:-minioadmin} + secret: ${MINIO_ROOT_PASSWORD:-minioadmin} + client_kwargs: + endpoint_url: "http://minio:9000" + experiment_tracker: + name: mlflow_compose + flavor: mlflow + type: experiment_tracker + configuration: + tracking_uri: "http://mlflow:5000" + tracking_username: admin + tracking_password: admin volumes: postgres_data: + minio_data: + frontend_html: diff --git a/docs/Docker-installation.md b/docs/Docker-installation.md index e57352427..98644f13e 100644 --- a/docs/Docker-installation.md +++ b/docs/Docker-installation.md @@ -2,44 +2,87 @@ ## Prerequisites -- Docker Desktop or Docker Engine + Docker Compose +- Docker Engine or Docker Desktop, with Docker Compose 2.23.1 or newer + (`docker compose version`) - Git - 4GB+ RAM -- A running fair-py-ops dev stack (kind cluster + ZenML + STAC API + MinIO + MLflow).( TODO : Add a single command up to set everything up) ## Quick start ```bash git clone https://github.com/hotosm/fAIr.git cd fAIr -docker compose up postgres -d -cp backend/.env.example backend/.env -# fill in real values, see backend/README.md for which vars are required -docker compose up api worker +cp env_example .env +docker compose up ``` -The `frontend` service runs under the `dev` profile (`docker compose --profile dev up frontend`). +Open . The first boot pulls images and seeds the model +catalog, so it takes a few minutes; later starts are quick. -## Default ports +This runs the API, the frontend, a background worker, and the full dependency +set: Postgres with PostGIS, MinIO, a STAC catalog, MLflow, and a ZenML server. +To run from source instead, run `docker compose build` first. -- 3500: Frontend -- 8000: API -- 5434: PostgreSQL (host) +## Verifying the installation -## Endpoints +`test.py` at the repository root walks the whole flow one request at a time: +area of interest, dataset build, training, promotion, prediction. -- Frontend: -- API root: -- OpenAPI schema: -- Swagger UI: -- ReDoc: -- Health probes: +```bash +uv run test.py +``` + +It prints each step as it passes and exits non-zero on the first failure. Point +it at a stack on other ports with `--api`, `--stac`, and `--minio`. + +## Services + +| Service | URL | Credentials | +| --- | --- | --- | +| fAIr frontend and API | | Bearer `dev-token` | +| Swagger UI | | | +| ReDoc | | | +| OpenAPI schema | | | +| Health probes | | | +| ZenML | | `default`, empty password | +| STAC | | | +| MLflow | | | +| MinIO console | | `minioadmin` / `minioadmin` | +| PostgreSQL | `localhost:5434` | `admin` / `password` | + +All v1 routes are under `/api/v1/`. Versioning uses DRF `NamespaceVersioning`, +so `request.version` is set per request and `/api/v2/` is one URL line away when +needed. + +Every published port is overridable, so the stack can coexist with other +services: -All v1 routes are under `/api/v1/`. Versioning uses DRF `NamespaceVersioning`, so `request.version` is set per request and `/api/v2/` is one URL line away when needed. +```bash +API_PORT=8100 POSTGRES_PORT=5544 docker compose up +``` + +The docker network name is pinned to `fair-net`, because the ZenML orchestrator +attaches training containers to it by name. Two copies of the stack on one host +need that name changed as well. ## Configuration -`backend/.env` is read by `pydantic-settings`. Required vars raise at boot if missing. See [backend/README.md](../backend/README.md) for a concise overview and `backend/.env.example` for the annotated source of truth. +The root `.env` is read by `pydantic-settings`; required vars raise at boot if +missing. `env_example` holds working defaults for the compose setup, using +compose service names as hosts. + +To run Django on the host against the containerised dependencies, use +`backend/env_example` instead, which points at `localhost` and the published +ports. See [backend/README.md](../backend/README.md) for every variable. + +## Notes + +Prediction results are returned as presigned MinIO URLs signed for the +in-network `minio` host, so they do not resolve from a browser. Add +`127.0.0.1 minio` to `/etc/hosts` to open them directly. + +Training runs spawn containers through the host Docker socket, which is mounted +into the worker. ## Help diff --git a/docs/Infra.md b/docs/Infra.md index e69de29bb..57ef766e4 100644 --- a/docs/Infra.md +++ b/docs/Infra.md @@ -0,0 +1,80 @@ +# Infrastructure & Deployment Flow + +Our standard deployment process for other apps is +[here](https://docs.hotosm.org/devops/deployment-process) + +fAIr differs slightly, because we have: +- Versioning of both software, as well as AI models. +- A dedicated dev instance EC2 for easier development with all components. + +Currently model development happens in the `fAIr-models` repo, but this +might eventually move to the `fAIr` monorepo. + +The model flow works like this: +- Each model dir has a `stac-item.json`. These point at the moving + `dev-inference` image tag, and only seed a STAC the first time it starts up + (on dev, or a brand new prod). +- After that the STAC database is the source of truth, updated through the + Django admin. +- A CI matrix workflow builds an image for each dir under `./models` when its + contents change, tagged with the git SHA. +- In the Django admin we give a SHA a version (`vX.Y.Z-rc.N`, then `vX.Y.Z`) + and register it in the STAC, pinned to the image digest ('rc' release candidates are used for staging, before full production tagging). +- A `BaseModel` table holds the model name and its status. The version details + live entirely in the STAC though. + +## Step 1: Development + +> [!NOTE] +> The Environment +> - Single EC2, lightweight k3s cluster. +> - Manually updated / synced with dev. +> - Model registration in STAC etc is all manual. + +1. Users work on models in development, versioned as `-dev` + with a specific SHA tag too. +2. Development model image (deps + code) is pushed to GHCR. +3. On the dev EC2 they run a script to update the **dev** STAC + and knative records. +4. Any changes to the frontend / API are manually synced to + the dev EC2 instance. +5. The dev model can be tested on the dev instance, using the + dev STAC, ZenML, knative services. + +## Step 2: Staging + +> [!NOTE] +> The Environment +> - Runs all the same components as production, but +> start up via PR from `staging` --> `main`. +> - The components run inside the `fair-staging` +> namespace of the Kubernetes cluster, under +> domain `https://stage.ai.hotosm.org`. +> - Does not run it's own `knative` controller, +> instead using the cluster-wide instance. + +1. When we want to stabilise and push out a **new model**, or **updates to the + API / website**, we use the staging setup. +2. First a PR must be raised on the fAIr repo from `staging` --> `main`. + This will set up `https://stage.ai.hotosm.org` with ZenML / STAC / + Knative registration. +3. On boot the **staging** STAC is seeded (read-only) from the current + **production** STAC, so it mirrors live. +4. CI has already built the model image, tagged by SHA. In the Django admin, + give that SHA a candidate version (`vX.Y.Z-rc.N`), register it in the + staging STAC, and test it. +5. Once it looks good, register the model in production (Step 3) before merging + the PR to `main`. Merging shuts the staging env down. + +## Step 3: Production + +> [!NOTE] +> The Environment +> - Runs through tagged releases on Github, where ArgoCD +> picks up the latest helm chart tag and deploys. + +1. A new tagged version is made from the latest `main` code. +2. This triggers a redeploy of the fAIr website / API. +3. In the production Django admin, give the tested SHA a release version + (`vX.Y.Z`), register its STAC item pinned to the digest, and make it live. + The image is already in GHCR, so it is available straight away. diff --git a/docs/deployment.md b/docs/deployment.md index ec6d76a6e..4505994cc 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -34,7 +34,7 @@ Make sure you have backups available in case things go wrong ! + fairpredictor (https://pypi.org/project/fairpredictor/) * Verify Docker images for fAIr production are built and deployed -* [worker](https://github.com/hotosm/fAIr/pkgs/container/fair-worker) , [api](https://github.com/hotosm/fAIr/pkgs/container/fair-api) & [offline-predictor](https://github.com/hotosm/fAIr/pkgs/container/fair-offline-predictor ) images should be built and pointed to latest release +* [worker](https://github.com/hotosm/fAIr/pkgs/container/fair/worker) , [api](https://github.com/hotosm/fAIr/pkgs/container/fair/api) & [offline-predictor](https://github.com/hotosm/fAIr/pkgs/container/fair-offline-predictor ) images should be built and pointed to latest release * Verify Docker image for [fairpredictor](https://github.com/hotosm/fairpredictor/pkgs/container/fairpredictor) is built and deployed - Now create new task definition for api , worker , predictor and prediction worker , Verify the env variable changes if there are any - Deploy the services diff --git a/docs/infra/dev.md b/docs/infra/dev.md new file mode 100644 index 000000000..fa54749c9 --- /dev/null +++ b/docs/infra/dev.md @@ -0,0 +1,123 @@ +# fAIr dev environment + +> Temporary setup. This Compose-on-EC2 deployment will be replaced by a k3s +> deployment once all the Helm charts are stable. + +The dev environment is a single EC2 instance running the whole fAIr stack with +Docker Compose, fronted by Caddy (automatic TLS). It tracks the `develop` +branch: CI builds the images on every push, and a redeploy pulls them. + +- App: https://dev.ai.hotosm.org (frontend + API, hanko login) +- STAC: https://stac.dev.ai.hotosm.org +- Access: `ssh fair-dev` + +## Layout on the box + +Everything lives in `/opt/fAIr-app` (a `develop` checkout): + +| File | Purpose | +|---|---| +| `docker-compose.yml` | base stack (api, worker, postgres, minio, stac, mlflow, zenml, frontend) | +| `docker-compose.dev.yml` | dev override: Caddy ingress, restart policies, the inline Caddyfile | +| `.env` | all runtime config and secrets (not in git) | + +`.env` sets `COMPOSE_FILE=docker-compose.yml:docker-compose.dev.yml`, so plain +`docker compose` commands pick up both files. The stack is managed by the +`fAIr-app` systemd unit and starts on boot. + +## Deploy / redeploy + +Pull the latest images and restart. Migrations run automatically on start. + +```bash +ssh fair-dev +cd /opt/fAIr-app +git pull +docker compose pull +sudo systemctl restart fAIr-app +``` + +## Start / stop / status + +Lifecycle is managed by systemd. + +```bash +sudo systemctl start fAIr-app +sudo systemctl stop fAIr-app +sudo systemctl status fAIr-app +``` + +## Logs + +Check container status and follow a service. Services: `api`, `worker`, +`caddy`, `stac`, `zenml`, `mlflow`, `postgres`, `minio`. + +```bash +cd /opt/fAIr-app +sudo docker compose ps +sudo docker compose logs -f api +``` + +## Editing config (`.env`) + +`.env` is the single source of truth, grouped into labeled blocks (Core, +Database, Auth, CORS, ZenML & STAC, Object storage, Frontend, Ports, Caddy). + +Apply changes by restarting (or recreate a single service with +`docker compose up -d `). + +```bash +sudo systemctl restart fAIr-app +``` + +Common changes: + +- **Django / API**: `DEBUG`, `SECRET_KEY`, `DATABASE_URL`, `CORS_ALLOWED_ORIGINS`. +- **Auth (hanko)**: `AUTH_PROVIDER`, `HANKO_API_URL`, `LOGIN_URL`, `COOKIE_*`, + `OSM_LOGIN_REDIRECT_URI`. +- **Frontend** (`VITE_*`, baked into `config.js` at container start): after any + change restart `api` too, it serves the SPA and caches `config.js` at boot. +- **Domain**: `PUBLIC_DOMAIN`, `FRONTEND_URL`, `API_BASE_URL`, `ALLOWED_HOSTS`, + `CSRF_TRUSTED_ORIGINS`, plus the domains in the Caddyfile. + +## Where to change what + +- **Service/image/port wiring**: `docker-compose.yml` (base) and + `docker-compose.dev.yml` (dev-only overrides). +- **Ingress, TLS, domains**: the `configs.caddyfile` block in + `docker-compose.dev.yml`. mlflow/zenml/minio subdomains are commented out + (kept internal); uncomment to expose them once DNS + auth are in place. +- **Runtime config / secrets**: `.env`. +- **Lifecycle**: `infra/systemd/fAIr-app.service`. + +## Access the database + +Postgres is bound to `127.0.0.1:5434` on the box. Open an SSH tunnel from your +machine. + +```bash +ssh -L 5434:localhost:5434 fair-dev +``` + +While the tunnel is open, connect locally with a client. Use the credentials +from `.env` (`POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB`). + +```bash +psql "postgresql://:@localhost:5434/" +``` + +## Internal services (mlflow, zenml, minio) + +Not exposed publicly. Tunnel to them: mlflow on `localhost:5000`, zenml on +`localhost:8080`, minio console on `localhost:9001`. + +```bash +ssh -L 5000:localhost:5000 -L 8080:localhost:8080 -L 9001:localhost:9001 fair-dev +``` + +## Notes + +- Training runs on the instance GPU (Docker default runtime is nvidia; ZenML + spawns the training container on the same Docker network). +- Dev data lives on the instance disk in Docker volumes; the shared EFS is not + used by dev. diff --git a/env_example b/env_example new file mode 100644 index 000000000..7f190d264 --- /dev/null +++ b/env_example @@ -0,0 +1,25 @@ +# cp env_example .env && docker compose up + + +DEBUG=true +SECRET_KEY=dev-secret-change-me +DATABASE_URL=postgis://admin:password@postgres:5432/fair +FRONTEND_URL=http://localhost:8000 +API_BASE_URL=http://localhost:8000/api/v1 + +# Static-token auth for local use +AUTH_PROVIDER=dev +FAIR_DEV_TOKEN=dev-token + +# ZENML_STORE_* are read by the zenml library itself; the server's default user +# has an empty password. Point the FAIR_* pair at a remote deployment to use one. +FAIR_ZENML_STORE_URL=http://zenml:8080 +FAIR_STAC_API_URL=http://stac:8080 +ZENML_STORE_URL=http://zenml:8080 +ZENML_STORE_USERNAME=default +ZENML_STORE_PASSWORD= + +BUCKET_NAME=fair-data +AWS_ENDPOINT_URL=http://minio:9000 +AWS_ACCESS_KEY_ID=minioadmin +AWS_SECRET_ACCESS_KEY=minioadmin diff --git a/frontend/.env.sample b/frontend/.env.sample index d3dd7dd3e..9cb470a6f 100644 --- a/frontend/.env.sample +++ b/frontend/.env.sample @@ -18,6 +18,11 @@ VITE_MATOMO_ID = 0 # Default value: "fair.hotosm.org". VITE_MATOMO_APP_DOMAIN = "fair.hotosm.org" +# The Matomo tracking URL. +# Data type: String (e.g., "https://matomo.hotosm.org"). +# Default value: "https://matomo.hotosm.org". +VITE_MATOMO_TRACKING_URL = "https://matomo.hotosm.org" + # The cache duration for polling the backend for updated statistics, in seconds. # Data type: Positive Integer (e.g., 900). # Default value: 900 seconds (15 minutes). @@ -181,12 +186,6 @@ VITE_OAM_TITILER_ENDPOINT = "https://titiler.hotosm.org/" # Default value: "https://oin-hotosm-temp.s3.us-east-1.amazonaws.com/". VITE_OAM_S3_BUCKET_URL = "https://oin-hotosm-temp.s3.us-east-1.amazonaws.com/" -# The Matomo tracking URL. -# Data type: String (e.g., "https://matomo.hotosm.org"). -# Default value: "https://matomo.hotosm.org". -VITE_MATOMO_TRACKING_URL = "https://matomo.hotosm.org" - - # The timeout duration for the invite banner, in milliseconds (ms). # Data type: Positive Integer (e.g., 3000). # Default value: 3000 milliseconds (3 seconds). @@ -265,7 +264,7 @@ VITE_HANKO_AUTH_TOKEN = "xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" # The base URL for the FAIR STAC Catalog. # Data type: String (e.g., "https://stac.fair.krschap.tech/"). # Default value: "https://stac.fair.krschap.tech/". -FAIR_STAC_CATALOG_BASE_URL = "https://stac.fair.krschap.tech/" +VITE_FAIR_STAC_CATALOG_BASE_URL = "https://stac.fair.krschap.tech/" # The environment mode for the application. # Data type: String (e.g., "development", "production"). diff --git a/frontend/Dockerfile.prod b/frontend/Dockerfile.prod index d79d22f41..efc6f3a18 100644 --- a/frontend/Dockerfile.prod +++ b/frontend/Dockerfile.prod @@ -1,17 +1,20 @@ -## docker build -t fair-frontend:latest -f Dockerfile.prod . && container_id=$(docker create fair-frontend:latest) && docker cp $container_id:/app/dist ./dist && docker rm $container_id - +# Generates runtime config, copies the SPA into the shared volume, and exits. # Build stage -FROM node:22 AS builder +FROM node:22-slim AS builder WORKDIR /app +RUN corepack enable && corepack prepare pnpm@9.8.0 --activate COPY package.json pnpm-lock.yaml ./ -# COPY .env ./.env -RUN npm install -g pnpm@9.8.0 RUN pnpm install --frozen-lockfile COPY . . +ARG VITE_BASE_API_URL="/api/v1/" +ENV VITE_BASE_API_URL=${VITE_BASE_API_URL} RUN pnpm run build # Export stage -FROM alpine:latest AS exporter +FROM alpine:3.20 WORKDIR /app COPY --from=builder /app/dist ./dist +COPY docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/frontend/docker-entrypoint.sh b/frontend/docker-entrypoint.sh new file mode 100644 index 000000000..fee38000d --- /dev/null +++ b/frontend/docker-entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +DIST_DIR="/app/dist" +OUT_DIR="/frontend_html" + +echo "Generating runtime ${DIST_DIR}/config.js from environment..." +{ + echo "// Generated at container start by docker-entrypoint.sh. Do not edit." + echo "window.__RUNTIME_CONFIG__ = {" + for name in $(env | grep -E '^(VITE_|FAIR_VIDEO_)' | cut -d= -f1 | sort); do + value=$(printenv "$name") + # Escape values for JavaScript strings. + escaped=$(printf '%s' "$value" | sed 's/\\/\\\\/g; s/"/\\"/g') + printf ' "%s": "%s",\n' "$name" "$escaped" + done + echo "};" +} > "${DIST_DIR}/config.js" + +echo "Copying SPA from ${DIST_DIR} --> ${OUT_DIR}" +cp -a "${DIST_DIR}/." "${OUT_DIR}/" + +echo "Frontend assets ready." diff --git a/frontend/index.html b/frontend/index.html index 984d73570..1e3e313c5 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -21,6 +21,9 @@ HOT fAIr + + + diff --git a/frontend/public/config.js b/frontend/public/config.js new file mode 100644 index 000000000..474fc6949 --- /dev/null +++ b/frontend/public/config.js @@ -0,0 +1,3 @@ +// Runtime config stub. +// Overwritten by docker-entrypoint.sh in production. +window.__RUNTIME_CONFIG__ = window.__RUNTIME_CONFIG__ || {}; diff --git a/frontend/src/components/auth/auth-modal.tsx b/frontend/src/components/auth/auth-modal.tsx index 548c021ef..ce4c1b858 100644 --- a/frontend/src/components/auth/auth-modal.tsx +++ b/frontend/src/components/auth/auth-modal.tsx @@ -34,7 +34,7 @@ export const AuthenticationModal = ({ !emailVerification ) { const returnTo = `${FRONTEND_URL}${location.pathname}${location.search}`; - window.location.href = `${HANKO_URL}?return_to=${encodeURIComponent(returnTo)}`; + window.location.href = `${HANKO_URL}/app?return_to=${encodeURIComponent(returnTo)}`; } }, [isOpen, callbackPage, emailVerification, location]); diff --git a/frontend/src/components/shared/protected-route.tsx b/frontend/src/components/shared/protected-route.tsx index 6e0a0eb65..25ea618ee 100644 --- a/frontend/src/components/shared/protected-route.tsx +++ b/frontend/src/components/shared/protected-route.tsx @@ -18,7 +18,7 @@ export const ProtectedRoute: React.FC = ({ children }) => { const handleLogin = () => { if (AUTH_PROVIDER === "hanko") { const returnTo = `${FRONTEND_URL}${location.pathname}${location.search}`; - window.location.href = `${HANKO_URL}?return_to=${encodeURIComponent(returnTo)}`; + window.location.href = `${HANKO_URL}/app?return_to=${encodeURIComponent(returnTo)}`; } else { /* * Set the `backgroundLocation` in location state so that when we open the authentication modal we still see the current page in the background. diff --git a/frontend/src/config/env.ts b/frontend/src/config/env.ts index 62935a517..f5dc870d9 100644 --- a/frontend/src/config/env.ts +++ b/frontend/src/config/env.ts @@ -1,111 +1,116 @@ -/** - * The environment variables. - */ +/** Environment variables, preferring runtime config over build config. */ + +declare global { + interface Window { + __RUNTIME_CONFIG__?: Record; + } +} + +const runtime = + (typeof window !== "undefined" && window.__RUNTIME_CONFIG__) || {}; +const env: Record = { ...import.meta.env }; +for (const [key, value] of Object.entries(runtime)) { + if (value) env[key] = value; +} + export const ENVS = { - BASE_API_URL: import.meta.env.VITE_BASE_API_URL, + BASE_API_URL: env.VITE_BASE_API_URL, - AUTH_PROVIDER: import.meta.env.VITE_AUTH_PROVIDER, - HANKO_URL: import.meta.env.VITE_HANKO_URL, + AUTH_PROVIDER: env.VITE_AUTH_PROVIDER, + HANKO_URL: env.VITE_HANKO_URL, - MATOMO_ID: import.meta.env.VITE_MATOMO_ID, + MATOMO_ID: env.VITE_MATOMO_ID, - MATOMO_APP_DOMAIN: import.meta.env.VITE_MATOMO_APP_DOMAIN, + MATOMO_APP_DOMAIN: env.VITE_MATOMO_APP_DOMAIN, - KPI_STATS_CACHE_TIME: import.meta.env.VITE_KPI_STATS_CACHE_TIME, + KPI_STATS_CACHE_TIME: env.VITE_KPI_STATS_CACHE_TIME, - MAX_TRAINING_AREA_SIZE: import.meta.env.VITE_MAX_TRAINING_AREA_SIZE, + MAX_TRAINING_AREA_SIZE: env.VITE_MAX_TRAINING_AREA_SIZE, - MIN_TRAINING_AREA_SIZE: import.meta.env.VITE_MIN_TRAINING_AREA_SIZE, + MIN_TRAINING_AREA_SIZE: env.VITE_MIN_TRAINING_AREA_SIZE, - MAX_TRAINING_AREA_UPLOAD_FILE_SIZE: import.meta.env - .VITE_MAX_TRAINING_AREA_UPLOAD_FILE_SIZE, + MAX_TRAINING_AREA_UPLOAD_FILE_SIZE: + env.VITE_MAX_TRAINING_AREA_UPLOAD_FILE_SIZE, - FAIR_VERSION: import.meta.env.VITE_FAIR_VERSION, + FAIR_VERSION: env.VITE_FAIR_VERSION, - OSM_HASHTAGS: import.meta.env.VITE_OSM_HASHTAGS, + OSM_HASHTAGS: env.VITE_OSM_HASHTAGS, - MAX_ZOOM_LEVEL: import.meta.env.VITE_MAX_ZOOM_LEVEL, + MAX_ZOOM_LEVEL: env.VITE_MAX_ZOOM_LEVEL, - MIN_ZOOM_LEVEL_FOR_START_MAPPING_PREDICTION: import.meta.env - .VITE_MIN_ZOOM_LEVEL_FOR_START_MAPPING_PREDICTION, + MIN_ZOOM_LEVEL_FOR_START_MAPPING_PREDICTION: + env.VITE_MIN_ZOOM_LEVEL_FOR_START_MAPPING_PREDICTION, - MIN_ZOOM_LEVEL_FOR_TRAINING_AREA_LABELS: import.meta.env - .VITE_MIN_ZOOM_LEVEL_FOR_TRAINING_AREA_LABELS, + MIN_ZOOM_LEVEL_FOR_TRAINING_AREA_LABELS: + env.VITE_MIN_ZOOM_LEVEL_FOR_TRAINING_AREA_LABELS, - TRAINING_AREAS_AOI_FILL_COLOR: import.meta.env - .VITE_TRAINING_AREAS_AOI_FILL_COLOR, + TRAINING_AREAS_AOI_FILL_COLOR: env.VITE_TRAINING_AREAS_AOI_FILL_COLOR, - TRAINING_AREAS_AOI_OUTLINE_COLOR: import.meta.env - .VITE_TRAINING_AREAS_AOI_OUTLINE_COLOR, + TRAINING_AREAS_AOI_OUTLINE_COLOR: env.VITE_TRAINING_AREAS_AOI_OUTLINE_COLOR, - TRAINING_AREAS_AOI_OUTLINE_WIDTH: import.meta.env - .VITE_TRAINING_AREAS_AOI_OUTLINE_WIDTH, + TRAINING_AREAS_AOI_OUTLINE_WIDTH: env.VITE_TRAINING_AREAS_AOI_OUTLINE_WIDTH, - TRAINING_AREAS_AOI_FILL_OPACITY: import.meta.env - .VITE_TRAINING_AREAS_AOI_FILL_OPACITY, + TRAINING_AREAS_AOI_FILL_OPACITY: env.VITE_TRAINING_AREAS_AOI_FILL_OPACITY, - TRAINING_AREAS_AOI_LABELS_FILL_OPACITY: import.meta.env - .VITE_TRAINING_AREAS_AOI_LABELS_FILL_OPACITY, + TRAINING_AREAS_AOI_LABELS_FILL_OPACITY: + env.VITE_TRAINING_AREAS_AOI_LABELS_FILL_OPACITY, - TRAINING_AREAS_AOI_LABELS_OUTLINE_WIDTH: import.meta.env - .VITE_TRAINING_AREAS_AOI_LABELS_OUTLINE_WIDTH, + TRAINING_AREAS_AOI_LABELS_OUTLINE_WIDTH: + env.VITE_TRAINING_AREAS_AOI_LABELS_OUTLINE_WIDTH, - TRAINING_AREAS_AOI_LABELS_FILL_COLOR: import.meta.env - .VITE_TRAINING_AREAS_AOI_LABELS_FILL_COLOR, + TRAINING_AREAS_AOI_LABELS_FILL_COLOR: + env.VITE_TRAINING_AREAS_AOI_LABELS_FILL_COLOR, - TRAINING_AREAS_MASK_FILL_COLOR: import.meta.env - .VITE_TRAINING_AREAS_MASK_FILL_COLOR, + TRAINING_AREAS_MASK_FILL_COLOR: env.VITE_TRAINING_AREAS_MASK_FILL_COLOR, - TRAINING_AREAS_AOI_LABELS_OUTLINE_COLOR: import.meta.env - .VITE_TRAINING_AREAS_AOI_LABELS_OUTLINE_COLOR, + TRAINING_AREAS_AOI_LABELS_OUTLINE_COLOR: + env.VITE_TRAINING_AREAS_AOI_LABELS_OUTLINE_COLOR, - JOSM_REMOTE_URL: import.meta.env.VITE_JOSM_REMOTE_URL, + JOSM_REMOTE_URL: env.VITE_JOSM_REMOTE_URL, - TRAINING_AREA_LABELS_FETCH_POOLING_INTERVAL_MS: import.meta.env - .VITE_TRAINING_AREA_LABELS_FETCH_POOLING_INTERVAL_MS, + TRAINING_AREA_LABELS_FETCH_POOLING_INTERVAL_MS: + env.VITE_TRAINING_AREA_LABELS_FETCH_POOLING_INTERVAL_MS, - OSM_LAST_UPDATED_POOLING_INTERVAL_MS: import.meta.env - .VITE_OSM_LAST_UPDATED_POOLING_INTERVAL_MS, + OSM_LAST_UPDATED_POOLING_INTERVAL_MS: + env.VITE_OSM_LAST_UPDATED_POOLING_INTERVAL_MS, - MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREA_LABELS: import.meta.env - .VITE_MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREA_LABELS, + MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREA_LABELS: + env.VITE_MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREA_LABELS, - MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREAS: import.meta.env - .VITE_MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREAS, + MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREAS: + env.VITE_MAX_GEOJSON_FILE_UPLOAD_FOR_TRAINING_AREAS, - MAX_ACCEPTABLE_POLYGON_IN_TRAINING_AREA_GEOJSON_FILE: import.meta.env - .VITE_MAX_ACCEPTABLE_POLYGON_IN_TRAINING_AREA_GEOJSON_FILE, + MAX_ACCEPTABLE_POLYGON_IN_TRAINING_AREA_GEOJSON_FILE: + env.VITE_MAX_ACCEPTABLE_POLYGON_IN_TRAINING_AREA_GEOJSON_FILE, - FAIR_PREDICTOR_API_URL: import.meta.env.VITE_FAIR_PREDICTOR_API_URL, + FAIR_PREDICTOR_API_URL: env.VITE_FAIR_PREDICTOR_API_URL, - OSM_DATABASE_STATUS_API_URL: import.meta.env.VITE_OSM_DATABASE_STATUS_API_URL, + OSM_DATABASE_STATUS_API_URL: env.VITE_OSM_DATABASE_STATUS_API_URL, - OAM_TITILER_ENDPOINT: import.meta.env.VITE_OAM_TITILER_ENDPOINT, + OAM_TITILER_ENDPOINT: env.VITE_OAM_TITILER_ENDPOINT, - OAM_S3_BUCKET_URL: import.meta.env.VITE_OAM_S3_BUCKET_URL, + OAM_S3_BUCKET_URL: env.VITE_OAM_S3_BUCKET_URL, - MATOMO_TRACKING_URL: import.meta.env.VITE_MATOMO_TRACKING_URL, - BANNER_TIMEOUT_DURATION: import.meta.env.VITE_BANNER_TIMEOUT_DURATION, - MAXIMUM_PREDICTION_AREA: import.meta.env.VITE_MAXIMUM_PREDICTION_AREA, - MAXIMUM_PREDICTION_TOLERANCE: import.meta.env - .VITE_MAXIMUM_PREDICTION_TOLERANCE, - FAIR_MODELS_BASE_PATH: import.meta.env.VITE_FAIR_MODELS_BASE_PATH, - OFFSET_STEP: import.meta.env.VITE_OFFSET_STEP, - MAXIMUM_ORTHO_SKEW_TOLEARANCE_AND_MAX_ANGLE_CHANGE_IN_DEGREES: import.meta.env - .VITE_MAXIMUM_ORTHO_SKEW_TOLEARANCE_AND_MAX_ANGLE_CHANGE_IN_DEGREES, + MATOMO_TRACKING_URL: env.VITE_MATOMO_TRACKING_URL, + BANNER_TIMEOUT_DURATION: env.VITE_BANNER_TIMEOUT_DURATION, + MAXIMUM_PREDICTION_AREA: env.VITE_MAXIMUM_PREDICTION_AREA, + MAXIMUM_PREDICTION_TOLERANCE: env.VITE_MAXIMUM_PREDICTION_TOLERANCE, + FAIR_MODELS_BASE_PATH: env.VITE_FAIR_MODELS_BASE_PATH, + OFFSET_STEP: env.VITE_OFFSET_STEP, + MAXIMUM_ORTHO_SKEW_TOLEARANCE_AND_MAX_ANGLE_CHANGE_IN_DEGREES: + env.VITE_MAXIMUM_ORTHO_SKEW_TOLEARANCE_AND_MAX_ANGLE_CHANGE_IN_DEGREES, - PREDICTIONS_RESULTS_POINT_FILL_COLOR: import.meta.env - .VITE_PREDICTIONS_RESULTS_POINT_FILL_COLOR, - PREDICTIONS_RESULTS_POINT_OUTLINE_COLOR: import.meta.env - .VITE_PREDICTIONS_RESULTS_POINT_OUTLINE_COLOR, + PREDICTIONS_RESULTS_POINT_FILL_COLOR: + env.VITE_PREDICTIONS_RESULTS_POINT_FILL_COLOR, + PREDICTIONS_RESULTS_POINT_OUTLINE_COLOR: + env.VITE_PREDICTIONS_RESULTS_POINT_OUTLINE_COLOR, - FAIR_YOUTUBE_UPDATES_URL: import.meta.env.FAIR_VIDEO_UPDATES_URL, - FAIR_VIDEO_BACKUP_URL: import.meta.env.FAIR_VIDEO_BACKUP_URL, - MAPSWIPE_VERIFICATION_NUMBER: import.meta.env - .VITE_MAPSWIPE_VERIFICATION_NUMBER, - MAPSWIPE_GROUP_SIZE: import.meta.env.VITE_MAPSWIPE_GROUP_SIZE, - FAIR_STAC_CATALOG_BASE_URL: import.meta.env.VITE_FAIR_STAC_CATALOG_BASE_URL, - NODE_ENV: import.meta.env.VITE_NODE_ENV, - TRY_FAIR_GRID_SIZE: import.meta.env.VITE_TRY_FAIR_GRID_SIZE, - FAIR_PROD_URL: import.meta.env.VITE_FAIR_PROD_URL, + FAIR_YOUTUBE_UPDATES_URL: env.FAIR_VIDEO_UPDATES_URL, + FAIR_VIDEO_BACKUP_URL: env.FAIR_VIDEO_BACKUP_URL, + MAPSWIPE_VERIFICATION_NUMBER: env.VITE_MAPSWIPE_VERIFICATION_NUMBER, + MAPSWIPE_GROUP_SIZE: env.VITE_MAPSWIPE_GROUP_SIZE, + FAIR_STAC_CATALOG_BASE_URL: env.VITE_FAIR_STAC_CATALOG_BASE_URL, + NODE_ENV: env.VITE_NODE_ENV, + TRY_FAIR_GRID_SIZE: env.VITE_TRY_FAIR_GRID_SIZE, + FAIR_PROD_URL: env.VITE_FAIR_PROD_URL, }; diff --git a/frontend/src/services/api-routes.ts b/frontend/src/services/api-routes.ts index bd1de02fd..6e9b72154 100644 --- a/frontend/src/services/api-routes.ts +++ b/frontend/src/services/api-routes.ts @@ -36,7 +36,7 @@ export const API_ENDPOINTS = { // KPIs - GET_KPI_STATS: "kpi/stats/ ", + GET_KPI_STATS: "kpi/stats/", // GeoJSON to OSM @@ -44,7 +44,7 @@ export const API_ENDPOINTS = { // Banner - GET_BANNER: "banner", + GET_BANNER: "banners/", // Models diff --git a/infra/systemd/fAIr-app.service b/infra/systemd/fAIr-app.service index 68ba6d5d0..366dcc7be 100644 --- a/infra/systemd/fAIr-app.service +++ b/infra/systemd/fAIr-app.service @@ -7,11 +7,8 @@ Requires=docker.service Type=oneshot RemainAfterExit=yes WorkingDirectory=/opt/fAIr-app -EnvironmentFile=/opt/fAIr-app/.env.production -User=root -Group=root -ExecStart=/usr/bin/docker compose -f /opt/fAIr-app/docker-compose.prod.yml --env-file /opt/fAIr-app/.env.production --profile gpu up --build -d -ExecStop=/usr/bin/docker compose -f /opt/fAIr-app/docker-compose.prod.yml down +ExecStart=/usr/bin/docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d +ExecStop=/usr/bin/docker compose -f docker-compose.yml -f docker-compose.dev.yml down Restart=on-failure RestartSec=30 diff --git a/infra/zenml/Dockerfile.cli b/infra/zenml/Dockerfile.cli index e573fd1ad..11144121b 100644 --- a/infra/zenml/Dockerfile.cli +++ b/infra/zenml/Dockerfile.cli @@ -2,7 +2,7 @@ ARG ZENML_VERSION FROM ghcr.io/spwoodcock/awscli-kubectl:latest AS cli-tools -FROM ghcr.io/hotosm/zenml-postgres:${ZENML_VERSION} AS cli +FROM ghcr.io/hotosm/fair/zenml-postgres:${ZENML_VERSION} AS cli USER root RUN apt-get update && apt-get install -y --no-install-recommends \ fish \ diff --git a/infra/zenml/compose.yml b/infra/zenml/compose.yml index 1e4bd0684..1bfa5f972 100644 --- a/infra/zenml/compose.yml +++ b/infra/zenml/compose.yml @@ -5,7 +5,7 @@ services: dockerfile: Dockerfile.postgres args: ZENML_VERSION: 0.94.1 - image: ghcr.io/hotosm/zenml-postgres:0.94.1 + image: ghcr.io/hotosm/fair/zenml-postgres:0.94.1 restart: on-failure ports: - "8080:8080" diff --git a/test.py b/test.py new file mode 100644 index 000000000..5b9a6e38b --- /dev/null +++ b/test.py @@ -0,0 +1,351 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = ["httpx>=0.27"] +# /// +"""End-to-end verification of the fAIr API against a running compose stack. + +uv run test.py --api http://localhost:8100 --stac http://localhost:8182 +""" + +from __future__ import annotations + +import argparse +import sys +import time +from typing import Any, Callable + +import httpx + +AOI_POLYGON = [ + [85.51678, 27.63133], + [85.52323, 27.63133], + [85.52323, 27.63743], + [85.51678, 27.63743], + [85.51678, 27.63133], +] +IMAGERY = ( + "https://tiles.openaerialmap.org/62d85d11d8499800053796c1/0" + "/62d85d11d8499800053796c2/{z}/{x}/{y}" +) +BASE_MODEL = "unet-segmentation" + + +class StepFailed(Exception): + """A check did not hold. Carries the human-readable reason.""" + + +def require(condition: object, message: str) -> None: + if not condition: + raise StepFailed(message) + + +def poll( + describe: Callable[[Any], str], + fetch: Callable[[], Any], + is_done: Callable[[Any], bool], + timeout_s: int, + interval_s: int = 5, +) -> Any: + """Call `fetch` until `is_done`, or raise once `timeout_s` elapses.""" + deadline = time.monotonic() + timeout_s + while True: + value = fetch() + if is_done(value): + return value + if time.monotonic() >= deadline: + raise StepFailed(f"still {describe(value)} after {timeout_s}s") + time.sleep(interval_s) + + +class Fair: + def __init__( + self, api_root: str, stac_root: str, minio_root: str, token: str + ) -> None: + self.api = f"{api_root.rstrip('/')}/api/v1" + self.stac = stac_root.rstrip("/") + self.minio = minio_root.rstrip("/") + self.http = httpx.Client( + headers={"Authorization": f"Bearer {token}"}, timeout=60.0 + ) + + def get(self, path: str, **kwargs: Any) -> Any: + return self._json(self.http.get(f"{self.api}{path}", **kwargs)) + + def post(self, path: str, payload: dict) -> Any: + return self._json(self.http.post(f"{self.api}{path}", json=payload)) + + @staticmethod + def _json(response: httpx.Response) -> Any: + if response.is_error: + raise StepFailed( + f"HTTP {response.status_code} {response.url}: {response.text[:200]}" + ) + return response.json() + + +def await_api(fair: Fair, ctx: dict) -> str: + """The api container accepts connections only once Django finishes booting.""" + + def probe() -> int | None: + try: + return fair.http.get(f"{fair.api}/health/").status_code + except httpx.TransportError: + return None + + poll( + describe=lambda code: "unreachable" if code is None else f"HTTP {code}", + fetch=probe, + is_done=lambda code: code is not None, + timeout_s=300, + interval_s=3, + ) + return f"{fair.api} accepting connections" + + +def check_health(fair: Fair, ctx: dict) -> str: + health = fair.get("/health/") + down = [ + name + for name in ("postgresql", "s3", "stac_api", "zenml") + if not health.get(name) + ] + require(not down, f"dependencies unreachable: {down}") + missing = [name for name, ok in health["stac_collections"].items() if not ok] + require(not missing, f"STAC collections missing: {missing}") + return "postgres, s3, stac, zenml and all 3 collections up" + + +def check_base_models(fair: Fair, ctx: dict) -> str: + items = fair.http.get( + f"{fair.stac}/collections/base-models/items", params={"limit": 50} + ) + names = sorted(f["id"] for f in Fair._json(items)["features"]) + require(names, "base-models collection is empty, stac-seed did not run") + require(BASE_MODEL in names, f"{BASE_MODEL} not seeded, found {names}") + return f"{len(names)} seeded: {', '.join(names)}" + + +def check_auth(fair: Fair, ctx: dict) -> str: + user = fair.get("/auth/me/") + require(user.get("osm_id"), f"no osm_id in {user}") + return f"osm_id={user['osm_id']} username={user['username']}" + + +def create_aoi(fair: Fair, ctx: dict) -> str: + aoi = fair.post( + "/aois/", + { + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": [AOI_POLYGON]}, + "properties": {"dataset": None}, + }, + ) + ctx["aoi_id"] = aoi["properties"]["id"] + return f"aoi id={ctx['aoi_id']}" + + +def build_dataset(fair: Fair, ctx: dict) -> str: + dataset = fair.post( + "/datasets/build/", + { + "title": f"e2e-banepa-{int(time.time())}", + "description": "end-to-end verification", + "source_imagery": IMAGERY, + "zoom": 19, + "aoi_ids": [ctx["aoi_id"]], + "label_tasks": ["semantic-segmentation"], + "label_classes": [{"name": "building", "classes": ["*"]}], + "keywords": ["building", "polygon"], + "label_type": "vector", + "geometry_type": "polygon", + }, + ) + ctx["dataset_id"] = dataset["id"] + ctx["dataset_stac_id"] = dataset["stac_id"] + return f"id={dataset['id']} stac_id={dataset['stac_id']} status={dataset['status']}" + + +def await_dataset(fair: Fair, ctx: dict) -> str: + dataset = poll( + describe=lambda d: d["status"], + fetch=lambda: fair.get(f"/datasets/{ctx['dataset_id']}/"), + is_done=lambda d: d["status"] in {"built", "failed"}, + timeout_s=600, + ) + require( + dataset["status"] == "built", + "dataset build failed, see `docker compose logs worker`", + ) + return "status=built, chips and labels uploaded" + + +def submit_training(fair: Fair, ctx: dict) -> str: + run = fair.post( + "/trainings/submit/", + { + "base_model_stac_id": BASE_MODEL, + "dataset_stac_id": ctx["dataset_stac_id"], + "model_name": f"e2e-unet-{int(time.time())}", + }, + ) + ctx["training_id"] = run["id"] + return f"id={run['id']} status={run['status']}" + + +def await_training(fair: Fair, ctx: dict) -> str: + run = poll( + describe=lambda r: r["status"], + fetch=lambda: fair.get(f"/trainings/{ctx['training_id']}/"), + is_done=lambda r: ( + r["status"] in {"completed", "failed", "stopped", "cached", "retried"} + ), + timeout_s=1800, + interval_s=15, + ) + require( + run["status"] == "completed", + f"training ended as {run['status']}, see `docker compose logs worker`", + ) + ctx["zenml_run_id"] = run["zenml_run_id"] + return f"status=completed zenml_run_id={run['zenml_run_id']}" + + +def check_run_endpoints(fair: Fair, ctx: dict) -> str: + run_id = ctx["zenml_run_id"] + status = fair.get(f"/trainings/runs/{run_id}/status/") + require(status["status"] == "completed", f"run status endpoint says {status}") + require(status["is_terminal"], "completed run not reported terminal") + logs = fair.get(f"/trainings/runs/{run_id}/logs/") + require(logs, "no log entries returned") + return f"status endpoint terminal, {len(logs)} log entries streamed" + + +def promote(fair: Fair, ctx: dict) -> str: + published = fair.post( + f"/trainings/{ctx['training_id']}/publish/", + {"description": "end-to-end verification", "title": "e2e promoted model"}, + ) + ctx["local_model_stac_id"] = published["local_model_stac_id"] + return f"local_model_stac_id={ctx['local_model_stac_id']}" + + +def check_promoted_item(fair: Fair, ctx: dict) -> str: + item_id = ctx["local_model_stac_id"] + response = fair.http.get(f"{fair.stac}/collections/local-models/items/{item_id}") + item = Fair._json(response) + assets = item["assets"] + for key in ("model", "checkpoint", "training-metrics"): + require(key in assets, f"promoted item missing '{key}' asset: {sorted(assets)}") + hyperparameters = item["properties"].get("mlm:hyperparameters") or {} + require(hyperparameters, "no mlm:hyperparameters recorded on the promoted item") + return f"v{item['properties']['version']}, {len(assets)} assets, {len(hyperparameters)} hyperparameters" + + +def submit_prediction(fair: Fair, ctx: dict) -> str: + prediction = fair.post( + "/predictions/submit/", + { + "model_stac_id": ctx["local_model_stac_id"], + "image_uri": IMAGERY, + "bbox": [85.51678, 27.63133, 85.52323, 27.63743], + "zoom": 19, + "params": {"confidence_threshold": 0.25}, + }, + ) + ctx["prediction_id"] = prediction["id"] + return f"id={prediction['id']} status={prediction['status']}" + + +def await_prediction(fair: Fair, ctx: dict) -> str: + prediction = poll( + describe=lambda p: f"{p['status']}/results_ready={p['results_ready']}", + fetch=lambda: fair.get(f"/predictions/{ctx['prediction_id']}/"), + is_done=lambda p: p["results_ready"] or p["status"] in {"failed", "stopped"}, + timeout_s=1200, + interval_s=10, + ) + require(prediction["results_ready"], f"prediction ended as {prediction['status']}") + return "status=completed results_ready=true" + + +def check_prediction_results(fair: Fair, ctx: dict) -> str: + results = fair.get(f"/predictions/{ctx['prediction_id']}/result/") + for key in ("geojson", "fgb", "pmtiles"): + require(key in results, f"missing '{key}' in {sorted(results)}") + + # Presigned URLs are signed for the in-network `minio` host. The fair-data + # bucket allows anonymous download, so read the object directly instead. + path = results["geojson"].split("?", 1)[0].split("/", 3)[3] + geojson = Fair._json(fair.http.get(f"{fair.minio}/{path}")) + features = geojson.get("features", []) + require(features, "prediction geojson has no features") + require( + geojson["features"][0]["geometry"]["type"] == "Polygon", + f"unexpected geometry {geojson['features'][0]['geometry']['type']}", + ) + return f"3 output formats, {len(features)} polygons in the geojson" + + +def check_list_endpoints(fair: Fair, ctx: dict) -> str: + counts = [] + for name in ("datasets", "local-models", "trainings", "predictions"): + payload = fair.get(f"/{name}/") + counts.append(f"{name}={payload.get('count', '?')}") + return ", ".join(counts) + + +STEPS: list[tuple[str, Callable[[Fair, dict], str]]] = [ + ("wait for API", await_api), + ("health", check_health), + ("base models seeded", check_base_models), + ("authentication", check_auth), + ("create AOI", create_aoi), + ("build dataset", build_dataset), + ("await dataset build", await_dataset), + ("submit training", submit_training), + ("await training", await_training), + ("run status and logs", check_run_endpoints), + ("promote to local model", promote), + ("promoted STAC item", check_promoted_item), + ("submit prediction", submit_prediction), + ("await prediction", await_prediction), + ("prediction results", check_prediction_results), + ("list endpoints", check_list_endpoints), +] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--api", default="http://localhost:8000") + parser.add_argument("--stac", default="http://localhost:8082") + parser.add_argument("--minio", default="http://localhost:9000") + parser.add_argument("--token", default="dev-token") + args = parser.parse_args() + + fair = Fair(args.api, args.stac, args.minio, args.token) + ctx: dict = {} + started = time.monotonic() + + for number, (name, run_step) in enumerate(STEPS, start=1): + label = f"[{number:2}/{len(STEPS)}] {name}" + print(f"{label} ...", flush=True) + step_started = time.monotonic() + try: + detail = run_step(fair, ctx) + except (StepFailed, httpx.TransportError) as failure: + print( + f"{label} FAILED after {time.monotonic() - step_started:.0f}s\n {failure}" + ) + return 1 + print( + f"{label} ok ({time.monotonic() - step_started:.0f}s)\n {detail}", + flush=True, + ) + + print(f"\nall {len(STEPS)} steps passed in {time.monotonic() - started:.0f}s") + return 0 + + +if __name__ == "__main__": + sys.exit(main())