CSV file upload

api/.docker-env.template

@@ -16,3 +16,10 @@ API_PORT=8000
 DB_PORT=5432
 
 API_PATH_PREFIX=
+
+UPLOAD_FORWARD_ENDPOINT=
+UPLOAD_FORWARD_ACCESS_KEY=
+UPLOAD_FORWARD_SECRET_KEY=
+
+IDP_ROOT_URL=
+IDP_API_URL=

api/src/api/app/apis/utils_api.py

@@ -0,0 +1,39 @@
+# coding: utf-8
+
+import logging
+from typing import List  # noqa: F401
+from datetime import date
+from pydantic import Field, StrictStr
+from typing import List, Optional
+from typing_extensions import Annotated
+from fastapi import (  # noqa: F401
+    APIRouter,
+    Body,
+    File,
+    Path,
+    Query,
+    Request,
+    UploadFile,
+)
+
+from app.controller.utils_controller import UtilsController
+
+
+router = APIRouter()
+controller = UtilsController()
+
+log = logging.getLogger('API.Utils')
+logging.basicConfig(level=logging.INFO)
+
+@router.post(
+    "/utils/share/casedata",
+    status_code=202,
+    tags=["Utils"],
+)
+async def validate_and_forward_shared_case_data(
+    request: Request,
+    file: UploadFile = File(None, description="csv file of case data to share with ESID")
+) -> None:
+    """Share Case Data with ESID."""
+    log.info(f'POST /utils/caseshare received...')
+    return await controller.handle_case_data_validation_upload(file, request.state)

api/src/api/app/controller/utils_controller.py

@@ -0,0 +1,121 @@
+# coding: utf-8
+
+import logging
+from typing import ClassVar, Dict, List, Tuple  # noqa: F401
+from datetime import datetime
+from json import dumps
+from pathlib import Path
+from pydantic import Field, StrictBytes, StrictFloat, StrictInt, StrictStr
+from typing import Any, Dict, List, Optional, Tuple, Union, Set
+from typing_extensions import Annotated
+from fastapi import HTTPException, UploadFile
+from starlette.datastructures import State
+from app.models.user_detail import UserDetail
+from core import config
+from functools import lru_cache
+from minio import Minio
+import os
+import requests
+
+log = logging.getLogger('API.Utils')
+logging.basicConfig(level=logging.INFO)
+
+class UtilsController:
+
+    async def handle_case_data_validation_upload(
+        self,
+        file: UploadFile,
+        request_state: State,
+    ) -> None:
+        """Validate the upladed file and forward it"""
+        # Check if actually a csv file
+        if not file or not file.filename or not file.filename.lower().endswith('.csv'):
+            raise HTTPException(
+                status_code=400,
+                detail='No CSV file sent'
+            )
+        # Check mime type
+        valid_content_types = ['text/csv', 'application/vnd.ms-excel']
+        if file.content_type not in valid_content_types:
+            raise HTTPException(
+                status_code=400,
+                detail=f"File has the wrong content type. Accepts {valid_content_types} but got '{file.content_type}'"
+            )
+        # Check first line
+        line = file.file.readline().decode(encoding='utf-8')
+        num_cols = len(line.split(';'))  # This assumes ';' is always used as separator
+        if num_cols != 76:  # This is also assumes the file always hass this magic number of columns
+            raise HTTPException(
+                status_code=400,
+                detail=f"File has the wrong amount of columns. Needs 76 but has '{num_cols}'"
+            )
+
+        # Validation successful, upload to minio bucket
+        lha_id: str = request_state.realm
+        # Get lha display name
+        lha_name = next((realm['displayName'] for realm in get_realms() if realm['realm'] == lha_id), '')
+        uploader: UserDetail = request_state.user
+
+        meta = {
+            'lha': {
+                'id': lha_id,
+                'name': lha_name,
+            },
+            'uploader': {
+                'id': uploader.userId,
+                'email': uploader.email,
+                'roles': uploader.role,
+            },
+            'upload_timestamp': datetime.now().isoformat(),
+        }
+
+        object_path_in_bucket = os.path.join("arrivals", lha_id, file.filename)
+        log.info(f'uploading \"{file.filename}\" into \"{object_path_in_bucket}\"')
+        log.info(f'meta info: {meta}')
+
+        client = create_minio_client()
+        # go to end of stream to read size
+        file.file.seek(0, os.SEEK_END)
+        size = file.file.tell()
+        # reset to start for upload
+        file.file.seek(0, 0)
+        try:
+            result = client.put_object(
+                bucket_name='private-lha-data',
+                object_name=object_path_in_bucket,
+                data=file.file,
+                length=size,
+                metadata=meta
+            )
+            log.info(f'created: {result.object_name}, etag: {result.etag}, version: {result.version_id}')
+        except Exception as ex:
+            log.warning(f'Unable to upload file: {ex}')
+            raise HTTPException(
+                status_code=500,
+                detail='An error occurred during file upload. Check the logs or contact an administrator.'
+            )
+
+        return None
+
+@lru_cache
+def get_realms() -> List[Any]:
+    """
+    Request realm list from IDP API
+    """
+    result_realms = requests.get(f'{str(config.IDP_API_URL)}/realms')
+    if result_realms.status_code != 200:
+        raise HTTPException(status_code=500, detail='IDP API unreachable to request realms')
+    return result_realms.json()
+
+
+@lru_cache
+def create_minio_client() -> Minio:
+    """
+    Create a Minio client to upload to a bucket
+    """
+    client = Minio(
+        endpoint=str(config.UPLOAD_FORWARD_ENDPOINT),
+        access_key=str(config.UPLOAD_FORWARD_ACCESS_KEY),
+        secret_key=str(config.UPLOAD_FORWARD_SECRET_KEY),
+    )
+    return client

api/src/api/app/models/__init__.py

@@ -16,4 +16,5 @@
 from .reduced_info import ReducedInfo
 from .reduced_scenario import ReducedScenario
 from .scenario import Scenario
-from .tagged import Tagged
+from .tagged import Tagged
+from .user_detail import UserDetail

api/src/api/core/config.py

@@ -1,6 +1,6 @@
 from databases import DatabaseURL
 from starlette.config import Config
-from starlette.datastructures import Secret
+from starlette.datastructures import Secret, URL
 
 config = Config(".env")
 
@@ -28,4 +28,10 @@
 )
 
 # OAuth2 settings
-IDP_ROOT_URL = config("IDP_ROOT_URL", cast=str, default="https://dev.lokiam.de")
+IDP_ROOT_URL = config("IDP_ROOT_URL", cast=URL)
+IDP_API_URL = config("IDP_API_URL", cast=URL)
+
+# Forward of uploaded case file settings
+UPLOAD_FORWARD_ENDPOINT = config("UPLOAD_FORWARD_ENDPOINT", cast=URL)
+UPLOAD_FORWARD_ACCESS_KEY = config("UPLOAD_FORWARD_ACCESS_KEY", cast=Secret)
+UPLOAD_FORWARD_SECRET_KEY = config("UPLOAD_FORWARD_SECRET_KEY", cast=Secret)

api/src/api/requirements.txt

@@ -19,3 +19,5 @@ aiofiles==23.1.0
 numpy==1.26.4
 pandas==1.5.3
 h5py==3.10.0
+minio==7.2.15
+requests==2.32.4

api/src/api/server.py

@@ -20,6 +20,7 @@
 from app.apis.nodes_api import router as NodesApiRouter
 from app.apis.parameter_definitions_api import router as ParameterDefinitionsApiRouter
 from app.apis.scenarios_api import router as ScenariosApiRouter
+from app.apis.utils_api import router as UtilsApiRouter
 from app.middlewares.authentication_middleware import AuthenticationMiddleware
 
 app = FastAPI(
@@ -40,13 +41,14 @@
 
 app.add_middleware(AuthenticationMiddleware)
 
+app.include_router(ScenariosApiRouter)
+app.include_router(ModelsApiRouter)
 app.include_router(CompartmentsApiRouter)
 app.include_router(GroupsApiRouter)
-app.include_router(InterventionsApiRouter)
-app.include_router(ModelsApiRouter)
-app.include_router(NodesApiRouter)
 app.include_router(ParameterDefinitionsApiRouter)
-app.include_router(ScenariosApiRouter)
+app.include_router(NodesApiRouter)
+app.include_router(InterventionsApiRouter)
+app.include_router(UtilsApiRouter)
 
 """
 @app.post("/test_celery/{message}")