misc: add mcp integration tests and increase test coverage

.github/workflows/test.yml

@@ -613,6 +613,97 @@ jobs:
         echo "=== API Server Logs ==="
         cat /tmp/api-server.log || echo "No API server log found"
 
+  test-integration:
+    runs-on: ubuntu-latest
+    env:
+      HINDSIGHT_API_LLM_PROVIDER: groq
+      HINDSIGHT_API_LLM_API_KEY: ${{ secrets.GROQ_API_KEY }}
+      HINDSIGHT_API_LLM_MODEL: openai/gpt-oss-20b
+      HINDSIGHT_API_URL: http://localhost:8888
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: ".python-version"
+
+    - name: Build API
+      working-directory: ./hindsight-api
+      run: uv build
+
+    - name: Install API dependencies
+      working-directory: ./hindsight-api
+      run: uv sync --no-install-project --index-strategy unsafe-best-match
+
+    - name: Install integration test dependencies
+      working-directory: ./hindsight-integration-tests
+      run: uv sync
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api
+      run: |
+        uv run python -c "
+        from sentence_transformers import SentenceTransformer, CrossEncoder
+        print('Downloading embedding model...')
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        print('Downloading cross-encoder model...')
+        CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+        print('Models downloaded successfully')
+        "
+
+    - name: Create .env file
+      run: |
+        cat > .env << EOF
+        HINDSIGHT_API_LLM_PROVIDER=${{ env.HINDSIGHT_API_LLM_PROVIDER }}
+        HINDSIGHT_API_LLM_API_KEY=${{ env.HINDSIGHT_API_LLM_API_KEY }}
+        HINDSIGHT_API_LLM_MODEL=${{ env.HINDSIGHT_API_LLM_MODEL }}
+        EOF
+
+    - name: Start API server
+      run: |
+        ./scripts/dev/start-api.sh > /tmp/api-server.log 2>&1 &
+        echo "Waiting for API server to be ready..."
+        for i in {1..60}; do
+          if curl -sf http://localhost:8888/health > /dev/null 2>&1; then
+            echo "API server is ready after ${i}s"
+            break
+          fi
+          if [ $i -eq 60 ]; then
+            echo "API server failed to start after 60s"
+            cat /tmp/api-server.log
+            exit 1
+          fi
+          sleep 1
+        done
+
+    - name: Run integration tests
+      working-directory: ./hindsight-integration-tests
+      run: uv run pytest tests/ -v
+
+    - name: Show API server logs
+      if: always()
+      run: |
+        echo "=== API Server Logs ==="
+        cat /tmp/api-server.log || echo "No API server log found"
+
   test-litellm-integration:
     runs-on: ubuntu-latest
 

docker/standalone/Dockerfile

@@ -125,7 +125,6 @@ FROM python:3.11-slim AS api-only
 
 WORKDIR /app
 
-# Install pg0 dependencies (procps provides 'kill' command needed by pg0)
 # Note: libicu version varies by Debian version - try common versions in order
 RUN apt-get update && apt-get install -y \
     curl \
@@ -138,7 +137,6 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir uv
 
-# Create non-root user (PostgreSQL cannot run as root)
 RUN useradd -m -s /bin/bash hindsight
 
 # Copy API with virtual environment from builder
@@ -148,18 +146,12 @@ COPY --from=api-builder /app/api /app/api
 COPY docker/standalone/start-all.sh /app/start-all.sh
 RUN chmod +x /app/start-all.sh
 
-# Create data directory for pg0 and set ownership
-RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
+RUN chown -R hindsight:hindsight /app
 
-# Switch to non-root user
 USER hindsight
 
-# Set PATH for hindsight user
 ENV PATH="/app/api/.venv/bin:${PATH}"
 
-# pg0 will download PostgreSQL binaries on first run
-ENV PG0_HOME=/home/hindsight/.pg0
-
 # Pre-download ML models to avoid runtime download (conditional)
 ARG PRELOAD_ML_MODELS
 RUN if [ "$PRELOAD_ML_MODELS" = "true" ]; then \
@@ -224,7 +216,7 @@ FROM python:3.11-slim AS standalone
 
 WORKDIR /app
 
-# Install Node.js, curl, uv, and pg0 dependencies (procps provides 'kill' command needed by pg0)
+# Install Node.js, curl, uv, and system dependencies
 # Note: libicu version varies by Debian version - try common versions in order
 RUN apt-get update && apt-get install -y \
     curl \
@@ -239,7 +231,6 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir uv
 
-# Create non-root user (PostgreSQL cannot run as root)
 RUN useradd -m -s /bin/bash hindsight
 
 # Copy API with virtual environment from builder
@@ -260,28 +251,12 @@ WORKDIR /app
 COPY docker/standalone/start-all.sh /app/start-all.sh
 RUN chmod +x /app/start-all.sh
 
-# Create data directory for pg0 and set ownership
-RUN mkdir -p /app/data && chown -R hindsight:hindsight /app
+RUN chown -R hindsight:hindsight /app
 
-# Switch to non-root user
 USER hindsight
 
-# Set PATH for hindsight user
 ENV PATH="/app/api/.venv/bin:${PATH}"
 
-# Pre-cache PostgreSQL binaries by starting/stopping pg0-embedded
-# Note: We use a temp instance just to download binaries, then delete instance data
-# to avoid stale port config. Only installation binaries are kept.
-ENV PG0_HOME=/home/hindsight/.pg0
-RUN /app/api/.venv/bin/python -c "\
-from pg0 import Pg0; \
-print('Pre-caching PostgreSQL binaries...'); \
-pg = Pg0(name='temp-cache', username='hindsight', password='hindsight', database='hindsight'); \
-pg.start(); \
-pg.stop(); \
-print('PostgreSQL binaries cached')" && \
-    rm -rf /home/hindsight/.pg0/instances || echo "Pre-download skipped"
-
 # Pre-download ML models to avoid runtime download (conditional)
 ARG PRELOAD_ML_MODELS
 RUN if [ "$PRELOAD_ML_MODELS" = "true" ]; then \

hindsight-api/hindsight_api/alembic/env.py

@@ -7,11 +7,10 @@
 import os
 from pathlib import Path
 
+from alembic import context
 from dotenv import load_dotenv
 from sqlalchemy import engine_from_config, pool
 
-from alembic import context
-
 # Import your models here
 from hindsight_api.models import Base
 

hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py

@@ -9,11 +9,10 @@
 from collections.abc import Sequence
 
 import sqlalchemy as sa
+from alembic import op
 from pgvector.sqlalchemy import Vector
 from sqlalchemy.dialects import postgresql
 
-from alembic import op
-
 # revision identifiers, used by Alembic.
 revision: str = "5a366d414dce"
 down_revision: str | Sequence[str] | None = None

hindsight-api/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py

@@ -9,9 +9,8 @@
 from collections.abc import Sequence
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = "b7c4d8e9f1a2"

hindsight-api/hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py

@@ -9,9 +9,8 @@
 from collections.abc import Sequence
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = "c8e5f2a3b4d1"

hindsight-api/hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py

@@ -12,7 +12,6 @@
 from collections.abc import Sequence
 
 import sqlalchemy as sa
-
 from alembic import context, op
 
 # revision identifiers, used by Alembic.

hindsight-api/hindsight_api/alembic/versions/rename_personality_to_disposition.py

@@ -9,9 +9,8 @@
 from collections.abc import Sequence
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import context, op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = "rename_personality"

hindsight-api/hindsight_api/api/__init__.py

@@ -94,7 +94,7 @@ async def chained_lifespan(app_instance: FastAPI):
 
         # Mount the MCP middleware
         app.mount(mcp_mount_path, mcp_app)
-        logger.info(f"MCP server enabled at {mcp_mount_path}/{{bank_id}}/mcp")
+        logger.info(f"MCP server enabled at {mcp_mount_path}/")
 
     return app
 

hindsight-api/hindsight_api/api/http.py

@@ -1607,7 +1607,7 @@ async def api_list_documents(
             raise HTTPException(status_code=500, detail=str(e))
 
     @app.get(
-        "/v1/default/banks/{bank_id}/documents/{document_id}",
+        "/v1/default/banks/{bank_id}/documents/{document_id:path}",
         response_model=DocumentResponse,
         summary="Get document details",
         description="Get a specific document including its original text",
@@ -1639,7 +1639,7 @@ async def api_get_document(
             raise HTTPException(status_code=500, detail=str(e))
 
     @app.get(
-        "/v1/default/chunks/{chunk_id}",
+        "/v1/default/chunks/{chunk_id:path}",
         response_model=ChunkResponse,
         summary="Get chunk details",
         description="Get a specific chunk by its ID",
@@ -1668,7 +1668,7 @@ async def api_get_chunk(chunk_id: str, request_context: RequestContext = Depends
             raise HTTPException(status_code=500, detail=str(e))
 
     @app.delete(
-        "/v1/default/banks/{bank_id}/documents/{document_id}",
+        "/v1/default/banks/{bank_id}/documents/{document_id:path}",
         response_model=DeleteDocumentResponse,
         summary="Delete a document",
         description="Delete a document and all its associated memory units and links.\n\n"
@@ -1999,9 +1999,7 @@ async def api_retain(
                 if item.document_id:
                     content_dict["document_id"] = item.document_id
                 if item.entities:
-                    content_dict["entities"] = [
-                        {"text": e.text, "type": e.type or "CONCEPT"} for e in item.entities
-                    ]
+                    content_dict["entities"] = [{"text": e.text, "type": e.type or "CONCEPT"} for e in item.entities]
                 contents.append(content_dict)
 
             if request.async_: