From 6b428f7563c379c662f9b0f2a14e2ea0f7a25b0a Mon Sep 17 00:00:00 2001 From: Ernesto Ruge Date: Thu, 26 Oct 2023 16:22:28 +0200 Subject: [PATCH] reintroduce linting --- .github/workflows/lint.yml | 43 ++++++++ Makefile | 4 +- webapp/prometheus_api/prometheus_handler.py | 71 ++++++++------ webapp/prometheus_api/prometheus_models.py | 7 +- webapp/prometheus_api/prometheus_rest_api.py | 1 - .../parking_sites/parking_sites_schema.py | 98 +++++++++---------- .../parking_site_import_xlsx_service.py | 1 + .../parking_site_import_xlsx_validator.py | 5 +- .../parking_site_generic_import_mapper.py | 1 + .../parking_site_generic_import_service.py | 7 +- .../tasks/generic_import_heartbeat_task.py | 1 + webapp/services/tasks/task_runner_service.py | 1 + 12 files changed, 150 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..69348fa --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,43 @@ +name: lint + +on: + push: + branches: + - '*' + pull_request: + branches: + - '*' + # make workflow "callable" by others + workflow_call: + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - '3' + steps: + - name: checkout + uses: actions/checkout@v4 + - name: setup Python v${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: pip install + run: pip install -r requirements.txt -r requirements-dev.txt + + - name: lint using ruff + # We could also use the official GitHub Actions integration. + # https://beta.ruff.rs/docs/usage/#github-action + # uses: chartboost/ruff-action@v1 + run: ruff --exclude webapp/converter --output-format github ./webapp + + - name: format using black + # We could also use the official GitHub Actions integration. + # https://black.readthedocs.io/en/stable/integrations/github_actions.html + # uses: uses: psf/black@stable + run: | + black --exclude webapp/converter -S --check --diff ./webapp diff --git a/Makefile b/Makefile index 31b9718..a2d7cf5 100644 --- a/Makefile +++ b/Makefile @@ -179,9 +179,9 @@ open-coverage: .PHONY: lint-fix lint-fix: $(FLASK_RUN) ruff --exclude webapp/converters --fix ./webapp - $(FLASK_RUN) black ./webapp + $(FLASK_RUN) black --exclude webapp/converter ./webapp .PHONY: lint-check lint-check: $(FLASK_RUN) ruff --exclude webapp/converter ./webapp - $(FLASK_RUN) black -S --check --diff webapp + $(FLASK_RUN) black --exclude webapp/converter -S --check --diff webapp diff --git a/webapp/prometheus_api/prometheus_handler.py b/webapp/prometheus_api/prometheus_handler.py index c076c2d..4a6011d 100644 --- a/webapp/prometheus_api/prometheus_handler.py +++ b/webapp/prometheus_api/prometheus_handler.py @@ -66,28 +66,38 @@ def get_metrics(self) -> str: if source.status in [SourceStatus.DISABLED, SourceStatus.PROVISIONED]: continue if source.static_data_updated_at: - last_static_update_metrics.metrics.append(SourceMetric( - source=source.uid, - value=int((datetime.now(tz=timezone.utc) - source.static_data_updated_at).total_seconds()), - )) + last_static_update_metrics.metrics.append( + SourceMetric( + source=source.uid, + value=int((datetime.now(tz=timezone.utc) - source.static_data_updated_at).total_seconds()), + ) + ) if source.realtime_data_updated_at: - last_realtime_update_metrics.metrics.append(SourceMetric( + last_realtime_update_metrics.metrics.append( + SourceMetric( + source=source.uid, + value=int((datetime.now(tz=timezone.utc) - source.realtime_data_updated_at).total_seconds()), + ) + ) + source_static_parking_site_error_count.metrics.append( + SourceMetric( source=source.uid, - value=int((datetime.now(tz=timezone.utc) - source.realtime_data_updated_at).total_seconds()), - )) - source_static_parking_site_error_count.metrics.append(SourceMetric( - source=source.uid, - value=source.static_parking_site_error_count, - )) - source_realtime_parking_site_error_count.metrics.append(SourceMetric( - source=source.uid, - value=source.realtime_parking_site_error_count, - )) - if source.status == SourceStatus.FAILED: - failed_sources.metrics.append(SourceMetric( + value=source.static_parking_site_error_count, + ) + ) + source_realtime_parking_site_error_count.metrics.append( + SourceMetric( source=source.uid, - value=1, - )) + value=source.realtime_parking_site_error_count, + ) + ) + if source.status == SourceStatus.FAILED: + failed_sources.metrics.append( + SourceMetric( + source=source.uid, + value=1, + ) + ) metrics = ( last_static_update_metrics.to_metrics() @@ -117,17 +127,20 @@ def get_parking_site_metrics(self) -> list[str]: ) for parking_site in parking_sites: - parking_site_static_capacity.metrics.append(ParkingSiteMetric( - parking_site=parking_site.original_uid, - source=parking_site.source.uid, - value=parking_site.capacity, - )) - if parking_site.has_realtime_data: - parking_site_realtime_capacity.metrics.append(ParkingSiteMetric( + parking_site_static_capacity.metrics.append( + ParkingSiteMetric( parking_site=parking_site.original_uid, source=parking_site.source.uid, - value=parking_site.realtime_capacity, - )) + value=parking_site.capacity, + ) + ) + if parking_site.has_realtime_data: + parking_site_realtime_capacity.metrics.append( + ParkingSiteMetric( + parking_site=parking_site.original_uid, + source=parking_site.source.uid, + value=parking_site.realtime_capacity, + ) + ) return parking_site_static_capacity.to_metrics() + parking_site_realtime_capacity.to_metrics() - diff --git a/webapp/prometheus_api/prometheus_models.py b/webapp/prometheus_api/prometheus_models.py index 020ab40..c259f38 100644 --- a/webapp/prometheus_api/prometheus_models.py +++ b/webapp/prometheus_api/prometheus_models.py @@ -41,7 +41,6 @@ class Metrics: metrics: List[BaseMetric] = field(default_factory=list) def to_metrics(self) -> List[str]: - return ( - [f'# HELP {self.identifier} {self.help}', f'# TYPE {self.identifier} {self.type.name}'] - + [metric.to_metric(self.identifier) for metric in self.metrics] - ) + return [f'# HELP {self.identifier} {self.help}', f'# TYPE {self.identifier} {self.type.name}'] + [ + metric.to_metric(self.identifier) for metric in self.metrics + ] diff --git a/webapp/prometheus_api/prometheus_rest_api.py b/webapp/prometheus_api/prometheus_rest_api.py index 7bb392f..94fe493 100644 --- a/webapp/prometheus_api/prometheus_rest_api.py +++ b/webapp/prometheus_api/prometheus_rest_api.py @@ -45,4 +45,3 @@ def get(self): response = make_response(response_string) response.mimetype = 'text/plain; version=0.0.4' return response - diff --git a/webapp/public_rest_api/parking_sites/parking_sites_schema.py b/webapp/public_rest_api/parking_sites/parking_sites_schema.py index 50ab8d0..c5b24e8 100644 --- a/webapp/public_rest_api/parking_sites/parking_sites_schema.py +++ b/webapp/public_rest_api/parking_sites/parking_sites_schema.py @@ -20,13 +20,13 @@ source_schema = JsonSchema( title='Source', properties={ - 'uid': StringField(minLength=1, maxLength=256), - 'name': StringField(maxLength=256, required=False), - 'public_url': UriField(maxLength=4096), - 'last_import': DateTimeField(required=False), - 'attribution_license': StringField(required=False), - 'attribution_contributor': StringField(maxLength=256, required=False), - 'attribution_url': StringField(maxLength=256, required=False), + 'uid': StringField(minLength=1, maxLength=256), + 'name': StringField(maxLength=256, required=False), + 'public_url': UriField(maxLength=4096), + 'last_import': DateTimeField(required=False), + 'attribution_license': StringField(required=False), + 'attribution_contributor': StringField(maxLength=256, required=False), + 'attribution_url': StringField(maxLength=256, required=False), }, ) @@ -37,48 +37,48 @@ parking_site_schema = JsonSchema( title='ParkingSite', properties={ - 'source_id': IntegerField(minimum=1), - 'original_uid': StringField(maxLength=256), - 'name': StringField(maxLength=256), - 'operator_name': StringField(maxLength=256, required=False), - 'public_url': UriField(maxLength=4096, required=False), - 'address': StringField(maxLength=512, required=False), - 'description': StringField(maxLength=4096, required=False), - 'type': EnumField(enum=ParkingSiteType), - 'max_stay': IntegerField(minimum=0, required=False), - 'has_lighting': BooleanField(required=False), - 'park_and_ride_type': ArrayField(items=EnumField(enum=ParkAndRideType), required=False), - 'is_supervised': BooleanField(required=False), - 'has_realtime_data': BooleanField(default=False), - 'static_data_updated_at': DateTimeField(), - 'realtime_opening_status': DateTimeField(required=False), - 'lat': DecimalField(precision=10, scale=7), - 'lon': DecimalField(precision=10, scale=7), - 'capacity': IntegerField(minimum=0), - 'capacity_disabled': IntegerField(minimum=0, required=False), - 'capacity_woman': IntegerField(minimum=0, required=False), - 'capacity_family': IntegerField(minimum=0, required=False), - 'capacity_charging': IntegerField(minimum=0, required=False), - 'capacity_carsharing': IntegerField(minimum=0, required=False), - 'capacity_truck': IntegerField(minimum=0, required=False), - 'capacity_bus': IntegerField(minimum=0, required=False), - 'realtime_capacity': IntegerField(minimum=0, required=False), - 'realtime_capacity_disabled': IntegerField(minimum=0, required=False), - 'realtime_capacity_woman': IntegerField(minimum=0, required=False), - 'realtime_capacity_family': IntegerField(minimum=0, required=False), - 'realtime_capacity_charging': IntegerField(minimum=0, required=False), - 'realtime_capacity_carsharing': IntegerField(minimum=0, required=False), - 'realtime_capacity_truck': IntegerField(minimum=0, required=False), - 'realtime_capacity_bus': IntegerField(minimum=0, required=False), - 'realtime_free_capacity': IntegerField(minimum=0, required=False), - 'realtime_free_capacity_disabled': IntegerField(minimum=0, required=False), - 'realtime_free_capacity_woman': IntegerField(minimum=0, required=False), - 'realtime_free_capacity_family': IntegerField(minimum=0, required=False), - 'realtime_free_capacity_charging': IntegerField(minimum=0, required=False), - 'realtime_free_capacity_carsharing': IntegerField(minimum=0, required=False), - 'realtime_free_capacity_truck': IntegerField(minimum=0, required=False), - 'realtime_free_capacity_bus': IntegerField(minimum=0, required=False), - 'opening_hours': StringField(maxLength=512, required=False, description='OSM opening_hours format'), + 'source_id': IntegerField(minimum=1), + 'original_uid': StringField(maxLength=256), + 'name': StringField(maxLength=256), + 'operator_name': StringField(maxLength=256, required=False), + 'public_url': UriField(maxLength=4096, required=False), + 'address': StringField(maxLength=512, required=False), + 'description': StringField(maxLength=4096, required=False), + 'type': EnumField(enum=ParkingSiteType), + 'max_stay': IntegerField(minimum=0, required=False), + 'has_lighting': BooleanField(required=False), + 'park_and_ride_type': ArrayField(items=EnumField(enum=ParkAndRideType), required=False), + 'is_supervised': BooleanField(required=False), + 'has_realtime_data': BooleanField(default=False), + 'static_data_updated_at': DateTimeField(), + 'realtime_opening_status': DateTimeField(required=False), + 'lat': DecimalField(precision=10, scale=7), + 'lon': DecimalField(precision=10, scale=7), + 'capacity': IntegerField(minimum=0), + 'capacity_disabled': IntegerField(minimum=0, required=False), + 'capacity_woman': IntegerField(minimum=0, required=False), + 'capacity_family': IntegerField(minimum=0, required=False), + 'capacity_charging': IntegerField(minimum=0, required=False), + 'capacity_carsharing': IntegerField(minimum=0, required=False), + 'capacity_truck': IntegerField(minimum=0, required=False), + 'capacity_bus': IntegerField(minimum=0, required=False), + 'realtime_capacity': IntegerField(minimum=0, required=False), + 'realtime_capacity_disabled': IntegerField(minimum=0, required=False), + 'realtime_capacity_woman': IntegerField(minimum=0, required=False), + 'realtime_capacity_family': IntegerField(minimum=0, required=False), + 'realtime_capacity_charging': IntegerField(minimum=0, required=False), + 'realtime_capacity_carsharing': IntegerField(minimum=0, required=False), + 'realtime_capacity_truck': IntegerField(minimum=0, required=False), + 'realtime_capacity_bus': IntegerField(minimum=0, required=False), + 'realtime_free_capacity': IntegerField(minimum=0, required=False), + 'realtime_free_capacity_disabled': IntegerField(minimum=0, required=False), + 'realtime_free_capacity_woman': IntegerField(minimum=0, required=False), + 'realtime_free_capacity_family': IntegerField(minimum=0, required=False), + 'realtime_free_capacity_charging': IntegerField(minimum=0, required=False), + 'realtime_free_capacity_carsharing': IntegerField(minimum=0, required=False), + 'realtime_free_capacity_truck': IntegerField(minimum=0, required=False), + 'realtime_free_capacity_bus': IntegerField(minimum=0, required=False), + 'opening_hours': StringField(maxLength=512, required=False, description='OSM opening_hours format'), }, ) diff --git a/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_service.py b/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_service.py index 7be5d7d..cf21fde 100644 --- a/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_service.py +++ b/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_service.py @@ -23,6 +23,7 @@ ImportDatasetException, ImportException, ) + from .parking_site_import_xlsx_validator import ParkingSiteInput diff --git a/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_validator.py b/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_validator.py index b28fcf6..27541eb 100644 --- a/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_validator.py +++ b/webapp/services/import_service/default_xlsx/parking_site_import_xlsx_validator.py @@ -11,16 +11,17 @@ AnyOfValidator, DecimalValidator, IntegerValidator, + NumericValidator, StringValidator, - UrlValidator, NumericValidator, + UrlValidator, ) from webapp.common.validation import ( ExcelNoneable, ExcelTimeValidator, ExtendedBooleanValidator, - NumberCastingStringValidator, GermanDurationIntegerValidator, + NumberCastingStringValidator, ) diff --git a/webapp/services/import_service/generic/parking_site_generic_import_mapper.py b/webapp/services/import_service/generic/parking_site_generic_import_mapper.py index 167c4f0..ce73b8e 100644 --- a/webapp/services/import_service/generic/parking_site_generic_import_mapper.py +++ b/webapp/services/import_service/generic/parking_site_generic_import_mapper.py @@ -7,6 +7,7 @@ from webapp.models import ParkingSite from webapp.models.parking_site import OpeningStatus, ParkingSiteType + from .parking_site_generic_import_validator import LotDataInput, LotInfoInput diff --git a/webapp/services/import_service/generic/parking_site_generic_import_service.py b/webapp/services/import_service/generic/parking_site_generic_import_service.py index 07485c9..d4c6471 100644 --- a/webapp/services/import_service/generic/parking_site_generic_import_service.py +++ b/webapp/services/import_service/generic/parking_site_generic_import_service.py @@ -23,6 +23,7 @@ ConverterMissingException, ImportDatasetException, ) + from .parking_site_generic_import_mapper import ParkingSiteGenericImportMapper from .parking_site_generic_import_validator import LotDataInput, LotInfoInput @@ -31,7 +32,7 @@ class ParkingSiteGenericImportService(BaseService): source_repository: SourceRepository parking_site_repository: ParkingSiteRepository - converters: dict[str, ] + converters: dict lot_info_validator = DataclassValidator(LotInfoInput) lot_data_validator = DataclassValidator(LotDataInput) @@ -100,7 +101,7 @@ def update_source_static(self, source_uid: str): return if getattr(lot_infos, 'lot_error_count', None) is not None: - source.realtime_parking_site_error_count = getattr(lot_infos, 'lot_error_count') + source.realtime_parking_site_error_count = lot_infos.lot_error_count for lot_info in lot_infos: try: @@ -165,7 +166,7 @@ def update_source_realtime(self, source_uid: str): return if getattr(lot_datasets, 'lot_error_count', None) is not None: - source.realtime_parking_site_error_count = getattr(lot_datasets, 'lot_error_count') + source.realtime_parking_site_error_count = lot_datasets.lot_error_count for lot_data in lot_datasets: try: diff --git a/webapp/services/tasks/generic_import_heartbeat_task.py b/webapp/services/tasks/generic_import_heartbeat_task.py index 746b44e..43cd8b3 100644 --- a/webapp/services/tasks/generic_import_heartbeat_task.py +++ b/webapp/services/tasks/generic_import_heartbeat_task.py @@ -6,6 +6,7 @@ from webapp.dependencies import dependencies from webapp.extensions import celery from webapp.services.import_service import ParkingSiteGenericImportService + from .base_task import BaseTask diff --git a/webapp/services/tasks/task_runner_service.py b/webapp/services/tasks/task_runner_service.py index 1516237..01ab40e 100644 --- a/webapp/services/tasks/task_runner_service.py +++ b/webapp/services/tasks/task_runner_service.py @@ -7,6 +7,7 @@ from webapp.common.config import ConfigHelper from webapp.common.logger import Logger from webapp.extensions import celery + from .base_task import BaseTask from .generic_import_heartbeat_task import RunGenericRealtimeImportTask