push support

Makefile

@@ -178,10 +178,10 @@ open-coverage:
 
 .PHONY: lint-fix
 lint-fix:
-	$(FLASK_RUN) ruff --exclude webapp/converters --fix ./webapp
-	$(FLASK_RUN) black --exclude webapp/converter ./webapp
+	$(FLASK_RUN) ruff --exclude webapp/converter --fix ./webapp
+	$(FLASK_RUN) black --exclude converter ./webapp
 
 .PHONY: lint-check
 lint-check:
 	$(FLASK_RUN) ruff --exclude webapp/converter ./webapp
-	$(FLASK_RUN) black --exclude webapp/converter -S --check --diff webapp
+	$(FLASK_RUN) black --exclude converter -S --check --diff webapp

docker-compose.yml

@@ -21,6 +21,8 @@ x-flask-defaults: &flask-defaults
   depends_on:
     postgresql:
       condition: service_healthy
+    mysql:
+      condition: service_healthy
     rabbitmq:
       condition: service_healthy
 

push-client/push-client.py

@@ -0,0 +1,68 @@
+"""
+Copyright 2023 binary butterfly GmbH
+Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
+"""
+
+import argparse
+import sys
+from getpass import getpass
+from pathlib import Path
+
+import requests
+
+DATA_TYPES = {
+    'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    'csv': 'text/csv',
+    'xml': 'application/xml',
+    'json': 'application/json',
+}
+
+PUSH_BASE_URL = 'http://localhost:5000/api/admin/v1/generic-parking-sites'
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog='ParkAPI Push Client',
+        description='This client helps to push static ParkAPI data',
+    )
+    parser.add_argument('source_uid')
+    parser.add_argument('file_path')
+    args = parser.parse_args()
+    source_uid: str = args.source_uid
+    file_path: Path = Path(args.file_path)
+
+    if not file_path.is_file():
+        sys.exit('Error: please add a file as second argument.')
+
+    password = getpass(f'Password for source UID {source_uid}: ')
+
+    file_ending = None
+    for ending in DATA_TYPES:
+        if file_path.name.endswith(f'.{ending}'):
+            file_ending = ending
+
+    if file_ending is None:
+        sys.exit(f'Error: invalid ending. Allowed endings are: {", ".join(DATA_TYPES.keys())}')
+
+    with file_path.open('rb') as file:
+        file_data = file.read()
+
+    endpoint = f'{PUSH_BASE_URL}/{file_ending}'
+    requests_response = requests.post(
+        url=endpoint,
+        data=file_data,
+        auth=(source_uid, password),
+        headers={'Content-Type': DATA_TYPES[file_ending]},
+    )
+
+    if requests_response.status_code == 204:
+        sys.exit('Upload successful.')
+
+    if requests_response.status_code == 401:
+        sys.exit('Access denied.')
+
+    sys.exit(f'Unknown error with HTTP status code {requests_response.status_code}.')
+
+
+if __name__ == "__main__":
+    main()

requirements-dev.txt

@@ -1,10 +1,3 @@
-pytest~=7.4.2
-pytest-cov~=4.1.0
+pytest~=7.4.3
 black~=23.10.0
-mypy~=1.6.1
-types-Flask-Migrate~=4.0.0.6
-types-PyYAML~=6.0.12.12
-types-requests~=2.31.0.10
-requests-mock~=1.11.0
-mypy-gitlab-code-quality~=1.0.0
-ruff~=0.1.0
+ruff~=0.1.3

requirements.txt

@@ -19,6 +19,7 @@ click~=8.1.7
 openpyxl~=3.1.2
 opening-hours-py~=0.6.17
 kombu~=5.3.2
+lxml~=4.9.3
 
 # required for converters
 beautifulsoup4~=4.12.2

webapp/admin_rest_api/generic_parking_sites/generic_parking_sites_handler.py

@@ -3,30 +3,111 @@
 Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
 """
 
+import csv
+from io import BytesIO, StringIO
+
+from lxml import etree
+from openpyxl.reader.excel import load_workbook
+
 from webapp.admin_rest_api import AdminApiBaseHandler
+from webapp.common.rest.exceptions import RestApiNotImplementedException
+from webapp.models import ParkingSite, Source
+from webapp.models.parking_site import ParkingSiteType
 from webapp.repositories import ParkingSiteRepository, SourceRepository
+from webapp.repositories.exceptions import ObjectNotFoundException
+from webapp.services.import_service import ParkingSiteGenericImportService
 
 
 class GenericParkingSitesHandler(AdminApiBaseHandler):
     source_repository: SourceRepository
     parking_site_repository: ParkingSiteRepository
+    parking_site_generic_import_service: ParkingSiteGenericImportService
 
     def __init__(
         self,
         *args,
         source_repository: SourceRepository,
         parking_site_repository: ParkingSiteRepository,
+        parking_site_generic_import_service: ParkingSiteGenericImportService,
         **kwargs,
     ):
         super().__init__(*args, **kwargs)
         self.source_repository = source_repository
         self.parking_site_repository = parking_site_repository
+        self.parking_site_generic_import_service = parking_site_generic_import_service
+
+    def handle_json_data(self, source_uid: str, data: dict | list):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        import_results = import_service.handle_json(data)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def handle_xml_data(self, source_uid: str, data: str):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        root_element = etree.parse(StringIO(data), parser=etree.XMLParser(resolve_entities=False))  # noqa: S320
+        import_results = import_service.handle_xml(root_element)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def handle_csv_data(self, source_uid: str, data: str):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        rows = list(csv.reader(StringIO(data)))
+        import_results = import_service.handle_csv(rows)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def handle_xlsx_data(self, source_uid: str, data: bytes):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        workbook = load_workbook(filename=BytesIO(data))
+        import_results = import_service.handle_xlsx(workbook)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def _get_source(self, source_uid: str) -> Source:
+        try:
+            source = self.source_repository.fetch_source_by_uid(source_uid)
+        except ObjectNotFoundException:
+            source = Source()
+            source.uid = source_uid
+            self.source_repository.save_source(source)
+
+        if source_uid not in self.parking_site_generic_import_service.push_converters:
+            raise RestApiNotImplementedException(message='Converter is missing for this source.')
+
+        return source
 
-    def handle_json_data(self, data: dict | list):
-        pass
+    def _save_parking_site_input(self, source: Source, static_parking_site_input):
+        try:
+            parking_site = self.parking_site_repository.fetch_parking_site_by_source_id_and_external_uid(
+                source_id=source.id,
+                original_uid=static_parking_site_input.uid,
+            )
+        except ObjectNotFoundException:
+            parking_site = ParkingSite()
+            parking_site.source_id = source.id
+            parking_site.original_uid = static_parking_site_input.uid
 
-    def handle_csv_data(self, data: bytes):
-        pass
+        for key, value in static_parking_site_input.to_dict().items():
+            if key in ['uid']:
+                continue
+            if key == 'type' and value:
+                value = ParkingSiteType[value.name]
+            setattr(parking_site, key, value)
 
-    def handle_xlsx_data(self, data: bytes):
-        pass
+        self.parking_site_repository.save_parking_site(parking_site)

webapp/admin_rest_api/generic_parking_sites/generic_parking_sites_rest_api.py

@@ -11,6 +11,7 @@
     GenericParkingSitesHandler,
 )
 from webapp.common.json import empty_json_response
+from webapp.common.server_auth import ServerAuthHelper
 from webapp.dependencies import dependencies
 
 
@@ -25,6 +26,7 @@ def __init__(self):
             **self.get_base_handler_dependencies(),
             parking_site_repository=dependencies.get_parking_site_repository(),
             source_repository=dependencies.get_source_repository(),
+            parking_site_generic_import_service=dependencies.get_parking_site_generic_import_service(),
         )
 
         self.add_url_rule(
@@ -78,13 +80,14 @@ class GenericParkingSitesJsonMethodView(GenericParkingSitesMethodView):
                     title='Generic JSON data',
                     properties={},
                     description='Any JSON data',
-                )
-            )
+                ),
+            ),
         ],
         response=[EmptyResponse(), ErrorResponse(error_codes=[400, 403])],
     )
     def post(self):
         self.generic_parking_sites_handler.handle_json_data(
+            source_uid=self.request_helper.get_basicauth_username(),
             data=self.request_helper.get_parsed_json(),
         )
 
@@ -98,8 +101,9 @@ class GenericParkingSitesXmlMethodView(GenericParkingSitesMethodView):
         response=[EmptyResponse(), ErrorResponse(error_codes=[400, 403])],
     )
     def post(self):
-        self.generic_parking_sites_handler.handle_json_data(
-            data=self.request_helper.get_parsed_json(),
+        self.generic_parking_sites_handler.handle_xml_data(
+            source_uid=self.request_helper.get_basicauth_username(),
+            data=self.request_helper.get_request_body_text(),
         )
 
         return empty_json_response(), 204
@@ -113,7 +117,8 @@ class GenericParkingSitesCsvMethodView(GenericParkingSitesMethodView):
     )
     def post(self):
         self.generic_parking_sites_handler.handle_csv_data(
-            data=self.request_helper.get_request_body(),
+            source_uid=self.request_helper.get_basicauth_username(),
+            data=self.request_helper.get_request_body_text(),
         )
 
         return empty_json_response(), 204
@@ -127,6 +132,7 @@ class GenericParkingSitesXlsxMethodView(GenericParkingSitesMethodView):
     )
     def post(self):
         self.generic_parking_sites_handler.handle_xlsx_data(
+            source_uid=self.request_helper.get_basicauth_username(),
             data=self.request_helper.get_request_body(),
         )
 

webapp/app.py

@@ -31,8 +31,8 @@ def launch(testing: bool = False) -> App:
         template_folder=os.path.join(BaseConfig.PROJECT_ROOT, 'templates'),
     )
     configure_app(app, testing=testing)
-    configure_blueprints(app)
     configure_extensions(app)
+    configure_blueprints(app)
     configure_logging(app)
     configure_error_handlers(app)
     configure_events(app)

webapp/common/rest/exceptions.py

@@ -42,42 +42,3 @@ class InputValidationException(RestApiException):
 class UnauthorizedException(RestApiException):
     code = 'unauthorized'
     http_status = 401
-
-
-class PatchWithoutChangesException(RestApiException):
-    """
-    The client sent a PATCH request with no data to be changed in the resource.
-
-    This happens either if the request body is a completely empty dictionary, or a non-empty dictionary that only contains fields not
-    recognized (and thus discarded) by the input validation (e.g. `{'foo': 123}` will be treated exactly like `{}` if the field "foo"
-    is not a defined field.
-
-    The latter can also happen due to user permissions in some cases (e.g. a different validataclass is used for limited operators,
-    where certain fields from the regular validataclass don't exist).
-    """
-
-    code = 'no_changes'
-    http_status = 400
-
-
-class ResourceInUseException(RestApiException):
-    """
-    The client tried to delete a resource that cannot be deleted currently because it is still in use (e.g. a pricegroup that is still
-    assigned to a unit).
-    """
-
-    code = 'resource_in_use'
-    http_status = 409
-
-
-class InvalidChildException(RestApiException):
-    """
-    The client referenced a resource in the context of a parent resource, but the referenced resource is not actually
-    a child of that parent resource.
-
-    For example, when patching a charge station, the connectors of that station can be patched in the same request.
-    This exception would be raised when the connectors do exist, but belong to a different station.
-    """
-
-    code = 'invalid_child'
-    http_status = 400

webapp/common/rest/request_helper.py

@@ -62,3 +62,12 @@ def get_origin(self) -> str:
 
     def get_request_body(self) -> bytes:
         return self.request.get_data()
+
+    def get_request_body_text(self) -> str:
+        return self.request.get_data(as_text=True)
+
+    def get_basicauth_username(self) -> Optional[str]:
+        if not self.request.authorization:
+            return None
+
+        return self.request.authorization.username

webapp/common/server_auth/server_auth_users.py

@@ -19,7 +19,7 @@ class ServerAuthRole(Enum):
     Roles a server API user can have.
     """
 
-    USER = 'user'
+    PUSH_CLIENT = 'push-client'
 
 
 @dataclass

webapp/dependencies.py

@@ -16,6 +16,7 @@
 from webapp.common.remote_helper import RemoteHelper
 from webapp.common.rest import RequestHelper
 from webapp.repositories import BaseRepository, ParkingSiteRepository, SourceRepository
+from webapp.services.import_service import ParkingSiteGenericImportService
 from webapp.services.sqlalchemy_service import SqlalchemyService
 
 if TYPE_CHECKING:
@@ -150,6 +151,14 @@ def get_sqlalchemy_service(self) -> SqlalchemyService:
             **self.get_base_service_dependencies(),
         )
 
+    @cache_dependency
+    def get_parking_site_generic_import_service(self) -> ParkingSiteGenericImportService:
+        return ParkingSiteGenericImportService(
+            source_repository=self.get_source_repository(),
+            parking_site_repository=self.get_parking_site_repository(),
+            **self.get_base_service_dependencies(),
+        )
+
     @cache_dependency
     def get_task_runner(self) -> 'TaskRunner':
         from webapp.services.tasks import TaskRunner