heartbeat, metrics endpoint

.github/workflows/lint.yml

@@ -0,0 +1,43 @@
+name: lint
+
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+  # make workflow "callable" by others
+  workflow_call:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - '3'
+    steps:
+    - name: checkout
+      uses: actions/checkout@v4
+    - name: setup Python v${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+
+    - name: pip install
+      run: pip install -r requirements.txt -r requirements-dev.txt
+
+    - name: lint using ruff
+      # We could also use the official GitHub Actions integration.
+      # https://beta.ruff.rs/docs/usage/#github-action
+      # uses: chartboost/ruff-action@v1
+      run: ruff --exclude webapp/converter --output-format github ./webapp
+
+    - name: format using black
+      # We could also use the official GitHub Actions integration.
+      # https://black.readthedocs.io/en/stable/integrations/github_actions.html
+      # uses: uses: psf/black@stable
+      run: |
+        black --exclude webapp/converter -S --check --diff ./webapp

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 /htmlcov
 /reports
 /*.sql
+/celerybeat-schedule.db
 
 /venv
 /logs/*

Makefile

@@ -1,4 +1,4 @@
-DOCKER_COMPOSE = docker compose
+DOCKER_COMPOSE = docker compose -f docker-compose.yml -f docker-compose.override.yml
 FLASK_RUN = $(DOCKER_COMPOSE) run --rm flask
 
 DOCKER_REGISTRY = ghcr.io
@@ -179,10 +179,9 @@ open-coverage:
 .PHONY: lint-fix
 lint-fix:
 	$(FLASK_RUN) ruff --exclude webapp/converters --fix ./webapp
-	$(FLASK_RUN) black ./webapp
+	$(FLASK_RUN) black --exclude webapp/converter ./webapp
 
 .PHONY: lint-check
 lint-check:
-
 	$(FLASK_RUN) ruff --exclude webapp/converter ./webapp
-	$(FLASK_RUN) black -S --check --diff webapp
+	$(FLASK_RUN) black --exclude webapp/converter -S --check --diff webapp

config_dist_dev.yaml

@@ -10,6 +10,8 @@ MAILS_FROM: '[email protected]'
 
 SECRET_KEY: 'random-key'
 SQLALCHEMY_DATABASE_URI: 'postgresql://park-api:development@postgresql/park-api'
+# Alternative: MariaDB / MySQL:
+# SQLALCHEMY_DATABASE_URI: 'mysql+pymysql://root:root@mysql/park-api'
 
 CELERY_BROKER_URL: 'amqp://rabbitmq'
 

dev/api_tests/public_api/park_api_v1.http

@@ -3,7 +3,7 @@ GET {{api_host}}/api/public/v1/park-api-v1
 
 
 ### List park api v1 detail data
-GET {{api_host}}/api/public/v1/park-api-v1/example-source
+GET {{api_host}}/api/public/v1/park-api-v1/p-r-vrs
 
 
 ### List park api v1 detail data with name filter

docker-compose.override.yml

@@ -0,0 +1,30 @@
+
+services:
+  mysql:
+    image: mariadb:11.1
+    volumes:
+      - .:/app
+      - mysql:/var/lib/mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+      MYSQL_DATABASE: park-api
+    healthcheck:
+      test: mariadb -h mysql -P 3306 -u root -p$$MYSQL_ROOT_PASSWORD -e "SELECT 1"
+      interval: 1s
+      timeout: 1s
+      retries: 20
+
+  phpmyadmin:
+    image: phpmyadmin/phpmyadmin:5.2
+    ports:
+      - "8081:80"
+    environment:
+      PMA_USER: root
+      PMA_PASSWORD: root
+      PMA_HOST: mysql
+      UPLOAD_LIMIT: 512M
+      # Disable spammy logging
+      APACHE_LOG_DIR: /tmp/logs
+
+volumes:
+  mysql:

docker-compose.yml

@@ -37,6 +37,14 @@ services:
     <<: *flask-defaults
     command: ["python3", "run-celery-dev.py"]
 
+  worker-heartbeat:
+    <<: *flask-defaults
+    command: ["python3", "run-celery-heartbeat-dev.py"]
+
+  flask-init-converters:
+    <<: *flask-defaults
+    command: ["flask", "source", "init-converters"]
+
   postgresql:
     image: postgis/postgis:16-3.4
     volumes:
@@ -64,5 +72,6 @@ services:
       timeout: 1s
       retries: 20
 
+
 volumes:
   postgresql:

migrations/versions/2023-10-25-12-33-52_687daced5384_init.py

@@ -0,0 +1,130 @@
+"""init
+
+Revision ID: 687daced5384
+Revises: 
+Create Date: 2023-10-25 12:33:52.521624
+
+"""
+import sqlalchemy as sa
+import sqlalchemy_utc
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+from webapp.common.sqlalchemy.point import Point
+
+# revision identifiers, used by Alembic.
+revision = '687daced5384'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('source',
+    sa.Column('uid', sa.String(length=256), nullable=False),
+    sa.Column('name', sa.String(length=256), nullable=True),
+    sa.Column('public_url', sa.String(length=4096), nullable=True),
+    sa.Column('static_data_updated_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=True),
+    sa.Column('realtime_data_updated_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=True),
+    sa.Column('attribution_license', sa.Text(), nullable=True),
+    sa.Column('attribution_contributor', sa.String(length=256), nullable=True),
+    sa.Column('attribution_url', sa.String(length=256), nullable=True),
+    sa.Column('status', sa.Enum('DISABLED', 'ACTIVE', 'FAILED', 'PROVISIONED', name='sourcestatus'), nullable=False),
+    sa.Column('static_parking_site_error_count', sa.Integer(), nullable=False),
+    sa.Column('realtime_parking_site_error_count', sa.Integer(), nullable=False),
+    sa.Column('id', sa.BigInteger(), nullable=False),
+    sa.Column('created_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=False),
+    sa.Column('modified_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_source')),
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_unicode_ci'
+    )
+    with op.batch_alter_table('source', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_source_created_at'), ['created_at'], unique=False)
+        batch_op.create_index(batch_op.f('ix_source_modified_at'), ['modified_at'], unique=False)
+
+    op.create_table('parking_site',
+    sa.Column('source_id', sa.BigInteger(), nullable=False),
+    sa.Column('original_uid', sa.String(length=256), nullable=False),
+    sa.Column('name', sa.String(length=256), nullable=False),
+    sa.Column('operator_name', sa.String(length=256), nullable=True),
+    sa.Column('public_url', sa.String(length=4096), nullable=True),
+    sa.Column('address', sa.String(length=512), nullable=True),
+    sa.Column('description', sa.String(length=4096), nullable=True),
+    sa.Column('type', sa.Enum('ON_STREET', 'OFF_STREET_PARKING_GROUND', 'UNDERGROUND', 'CAR_PARK', 'OTHER', name='parkingsitetype'), nullable=True),
+    sa.Column('max_stay', sa.Integer(), nullable=True),
+    sa.Column('has_lighting', sa.Boolean(), nullable=True),
+    sa.Column('fee_description', sa.String(length=256), nullable=True),
+    sa.Column('has_fee', sa.Boolean(), nullable=True),
+    sa.Column('park_and_ride_type', sa.String(length=256), nullable=True),
+    sa.Column('is_supervised', sa.Boolean(), nullable=True),
+    sa.Column('has_realtime_data', sa.Boolean(), nullable=False),
+    sa.Column('static_data_updated_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=True),
+    sa.Column('realtime_data_updated_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=True),
+    sa.Column('realtime_opening_status', sa.Enum('OPEN', 'CLOSED', 'UNKNOWN', name='openingstatus'), nullable=False),
+    sa.Column('lat', sa.Numeric(precision=10, scale=7), nullable=False),
+    sa.Column('lon', sa.Numeric(precision=10, scale=7), nullable=False),
+    sa.Column('capacity', sa.Integer(), nullable=True),
+    sa.Column('capacity_disabled', sa.Integer(), nullable=True),
+    sa.Column('capacity_woman', sa.Integer(), nullable=True),
+    sa.Column('capacity_family', sa.Integer(), nullable=True),
+    sa.Column('capacity_charging', sa.Integer(), nullable=True),
+    sa.Column('capacity_carsharing', sa.Integer(), nullable=True),
+    sa.Column('capacity_truck', sa.Integer(), nullable=True),
+    sa.Column('capacity_bus', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity_disabled', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity_woman', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity_family', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity_charging', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity_carsharing', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity_truck', sa.Integer(), nullable=True),
+    sa.Column('realtime_capacity_bus', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity_disabled', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity_woman', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity_family', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity_charging', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity_carsharing', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity_truck', sa.Integer(), nullable=True),
+    sa.Column('realtime_free_capacity_bus', sa.Integer(), nullable=True),
+    sa.Column('opening_hours', sa.String(length=512), nullable=True),
+    sa.Column('geometry', Point(), nullable=False),
+    sa.Column('id', sa.BigInteger(), nullable=False),
+    sa.Column('created_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=False),
+    sa.Column('modified_at', sqlalchemy_utc.sqltypes.UtcDateTime(timezone=True), nullable=False),
+    sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_parking_site_source_id')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_parking_site'))
+    )
+    with op.batch_alter_table('parking_site', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_parking_site_created_at'), ['created_at'], unique=False)
+        batch_op.create_index(batch_op.f('ix_parking_site_modified_at'), ['modified_at'], unique=False)
+        batch_op.create_index(batch_op.f('ix_parking_site_original_uid'), ['original_uid'], unique=False)
+        batch_op.create_index('ix_parking_site_source_original_uid', ['source_id', 'original_uid'], unique=True)
+    # ### end Alembic commands ###
+    engine_name = op.get_bind().engine.name
+    if engine_name == 'postgresql':
+        op.execute('CREATE INDEX ix_geometry_index ON parking_site USING GIST (geometry);')
+    elif engine_name == 'mysql':
+        op.execute('CREATE SPATIAL INDEX ix_geometry_index ON parking_site (geometry);')
+    else:
+        raise NotImplementedError('The application just supports mysql, mariadb and postgresql.')
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('parking_site', schema=None) as batch_op:
+        batch_op.drop_index('ix_parking_site_source_original_uid')
+        batch_op.drop_index(batch_op.f('ix_parking_site_original_uid'))
+        batch_op.drop_index(batch_op.f('ix_parking_site_modified_at'))
+        batch_op.drop_index(batch_op.f('ix_parking_site_created_at'))
+        batch_op.drop_index(batch_op.f('ix_geometry_index'))
+
+    op.drop_table('parking_site')
+    with op.batch_alter_table('source', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_source_modified_at'))
+        batch_op.drop_index(batch_op.f('ix_source_created_at'))
+
+    op.drop_table('source')
+    # ### end Alembic commands ###