diff --git a/api/.docker-env.template b/api/.docker-env.template index 1ff8af4..8c7ea8f 100644 --- a/api/.docker-env.template +++ b/api/.docker-env.template @@ -16,3 +16,10 @@ API_PORT=8000 DB_PORT=5432 API_PATH_PREFIX= + +UPLOAD_FORWARD_ENDPOINT= +UPLOAD_FORWARD_ACCESS_KEY= +UPLOAD_FORWARD_SECRET_KEY= + +IDP_ROOT_URL= +IDP_API_URL= \ No newline at end of file diff --git a/api/src/api/app/apis/utils_api.py b/api/src/api/app/apis/utils_api.py new file mode 100644 index 0000000..d00d50b --- /dev/null +++ b/api/src/api/app/apis/utils_api.py @@ -0,0 +1,39 @@ +# coding: utf-8 + +import logging +from typing import List # noqa: F401 +from datetime import date +from pydantic import Field, StrictStr +from typing import List, Optional +from typing_extensions import Annotated +from fastapi import ( # noqa: F401 + APIRouter, + Body, + File, + Path, + Query, + Request, + UploadFile, +) + +from app.controller.utils_controller import UtilsController + + +router = APIRouter() +controller = UtilsController() + +log = logging.getLogger('API.Utils') +logging.basicConfig(level=logging.INFO) + +@router.post( + "/utils/share/casedata", + status_code=202, + tags=["Utils"], +) +async def validate_and_forward_shared_case_data( + request: Request, + file: UploadFile = File(None, description="csv file of case data to share with ESID") +) -> None: + """Share Case Data with ESID.""" + log.info(f'POST /utils/caseshare received...') + return await controller.handle_case_data_validation_upload(file, request.state) \ No newline at end of file diff --git a/api/src/api/app/controller/utils_controller.py b/api/src/api/app/controller/utils_controller.py new file mode 100644 index 0000000..8d49ab4 --- /dev/null +++ b/api/src/api/app/controller/utils_controller.py @@ -0,0 +1,121 @@ +# coding: utf-8 + +import logging +from typing import ClassVar, Dict, List, Tuple # noqa: F401 +from datetime import datetime +from json import dumps +from pathlib import Path +from pydantic import Field, StrictBytes, StrictFloat, StrictInt, StrictStr +from typing import Any, Dict, List, Optional, Tuple, Union, Set +from typing_extensions import Annotated +from fastapi import HTTPException, UploadFile +from starlette.datastructures import State +from app.models.user_detail import UserDetail +from core import config +from functools import lru_cache +from minio import Minio +import os +import requests + +log = logging.getLogger('API.Utils') +logging.basicConfig(level=logging.INFO) + +class UtilsController: + + async def handle_case_data_validation_upload( + self, + file: UploadFile, + request_state: State, + ) -> None: + """Validate the upladed file and forward it""" + # Check if actually a csv file + if not file or not file.filename or not file.filename.lower().endswith('.csv'): + raise HTTPException( + status_code=400, + detail='No CSV file sent' + ) + # Check mime type + valid_content_types = ['text/csv', 'application/vnd.ms-excel'] + if file.content_type not in valid_content_types: + raise HTTPException( + status_code=400, + detail=f"File has the wrong content type. Accepts {valid_content_types} but got '{file.content_type}'" + ) + # Check first line + line = file.file.readline().decode(encoding='utf-8') + num_cols = len(line.split(';')) # This assumes ';' is always used as separator + if num_cols != 76: # This is also assumes the file always hass this magic number of columns + raise HTTPException( + status_code=400, + detail=f"File has the wrong amount of columns. Needs 76 but has '{num_cols}'" + ) + + # Validation successful, upload to minio bucket + lha_id: str = request_state.realm + # Get lha display name + lha_name = next((realm['displayName'] for realm in get_realms() if realm['realm'] == lha_id), '') + uploader: UserDetail = request_state.user + + meta = { + 'lha': { + 'id': lha_id, + 'name': lha_name, + }, + 'uploader': { + 'id': uploader.userId, + 'email': uploader.email, + 'roles': uploader.role, + }, + 'upload_timestamp': datetime.now().isoformat(), + } + + object_path_in_bucket = os.path.join("arrivals", lha_id, file.filename) + log.info(f'uploading \"{file.filename}\" into \"{object_path_in_bucket}\"') + log.info(f'meta info: {meta}') + + client = create_minio_client() + # go to end of stream to read size + file.file.seek(0, os.SEEK_END) + size = file.file.tell() + # reset to start for upload + file.file.seek(0, 0) + try: + result = client.put_object( + bucket_name='private-lha-data', + object_name=object_path_in_bucket, + data=file.file, + length=size, + metadata=meta + ) + log.info(f'created: {result.object_name}, etag: {result.etag}, version: {result.version_id}') + except Exception as ex: + log.warning(f'Unable to upload file: {ex}') + raise HTTPException( + status_code=500, + detail='An error occurred during file upload. Check the logs or contact an administrator.' + ) + + return None + +@lru_cache +def get_realms() -> List[Any]: + """ + Request realm list from IDP API + """ + result_realms = requests.get(f'{str(config.IDP_API_URL)}/realms') + if result_realms.status_code != 200: + raise HTTPException(status_code=500, detail='IDP API unreachable to request realms') + return result_realms.json() + + +@lru_cache +def create_minio_client() -> Minio: + """ + Create a Minio client to upload to a bucket + """ + client = Minio( + endpoint=str(config.UPLOAD_FORWARD_ENDPOINT), + access_key=str(config.UPLOAD_FORWARD_ACCESS_KEY), + secret_key=str(config.UPLOAD_FORWARD_SECRET_KEY), + ) + return client \ No newline at end of file diff --git a/api/src/api/app/models/__init__.py b/api/src/api/app/models/__init__.py index b3f0d0e..9258168 100644 --- a/api/src/api/app/models/__init__.py +++ b/api/src/api/app/models/__init__.py @@ -16,4 +16,5 @@ from .reduced_info import ReducedInfo from .reduced_scenario import ReducedScenario from .scenario import Scenario -from .tagged import Tagged \ No newline at end of file +from .tagged import Tagged +from .user_detail import UserDetail \ No newline at end of file diff --git a/api/src/api/core/config.py b/api/src/api/core/config.py index 0ab3b0e..023f1d7 100644 --- a/api/src/api/core/config.py +++ b/api/src/api/core/config.py @@ -1,6 +1,6 @@ from databases import DatabaseURL from starlette.config import Config -from starlette.datastructures import Secret +from starlette.datastructures import Secret, URL config = Config(".env") @@ -28,4 +28,10 @@ ) # OAuth2 settings -IDP_ROOT_URL = config("IDP_ROOT_URL", cast=str, default="https://dev.lokiam.de") \ No newline at end of file +IDP_ROOT_URL = config("IDP_ROOT_URL", cast=URL) +IDP_API_URL = config("IDP_API_URL", cast=URL) + +# Forward of uploaded case file settings +UPLOAD_FORWARD_ENDPOINT = config("UPLOAD_FORWARD_ENDPOINT", cast=URL) +UPLOAD_FORWARD_ACCESS_KEY = config("UPLOAD_FORWARD_ACCESS_KEY", cast=Secret) +UPLOAD_FORWARD_SECRET_KEY = config("UPLOAD_FORWARD_SECRET_KEY", cast=Secret) diff --git a/api/src/api/requirements.txt b/api/src/api/requirements.txt index cf4ebf0..c8a6d6a 100644 --- a/api/src/api/requirements.txt +++ b/api/src/api/requirements.txt @@ -19,3 +19,5 @@ aiofiles==23.1.0 numpy==1.26.4 pandas==1.5.3 h5py==3.10.0 +minio==7.2.15 +requests==2.32.4 \ No newline at end of file diff --git a/api/src/api/server.py b/api/src/api/server.py index 9a2642e..7070a26 100644 --- a/api/src/api/server.py +++ b/api/src/api/server.py @@ -20,6 +20,7 @@ from app.apis.nodes_api import router as NodesApiRouter from app.apis.parameter_definitions_api import router as ParameterDefinitionsApiRouter from app.apis.scenarios_api import router as ScenariosApiRouter +from app.apis.utils_api import router as UtilsApiRouter from app.middlewares.authentication_middleware import AuthenticationMiddleware app = FastAPI( @@ -40,13 +41,14 @@ app.add_middleware(AuthenticationMiddleware) +app.include_router(ScenariosApiRouter) +app.include_router(ModelsApiRouter) app.include_router(CompartmentsApiRouter) app.include_router(GroupsApiRouter) -app.include_router(InterventionsApiRouter) -app.include_router(ModelsApiRouter) -app.include_router(NodesApiRouter) app.include_router(ParameterDefinitionsApiRouter) -app.include_router(ScenariosApiRouter) +app.include_router(NodesApiRouter) +app.include_router(InterventionsApiRouter) +app.include_router(UtilsApiRouter) """ @app.post("/test_celery/{message}")