Merge pull request #499 from Police-Data-Accessibility-Project/mc_494_suggest_data_source

Begin draft

Author: maxachis

alembic/versions/2025_10_20_1620-6adf9d894180_enable_data_source_agency_submission.py

@@ -0,0 +1,109 @@
+"""Enable data source/agency submission
+
+Revision ID: 6adf9d894180
+Revises: 9d57b3b79d35
+Create Date: 2025-10-20 16:20:44.081736
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import ENUM, ARRAY
+
+
+# revision identifiers, used by Alembic.
+revision: str = '6adf9d894180'
+down_revision: Union[str, None] = '9d57b3b79d35'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+def upgrade() -> None:
+    _add_autogenerated_agency_id()
+    _add_new_columns_to_optional_ds_metadata()
+
+def _add_new_columns_to_optional_ds_metadata():
+    table_name: str = "url_optional_data_source_metadata"
+
+    agency_aggregation_enum = ENUM(
+        'federal',
+        'state',
+        'county',
+        'local',
+        name='agency_aggregation_enum',
+        create_type=True,
+    )
+    agency_aggregation_enum.create(op.get_bind())
+
+    update_method_enum = ENUM(
+        'Overwrite',
+        'Insert',
+        'No updates',
+        name='update_method_enum',
+        create_type=True
+    )
+    update_method_enum.create(op.get_bind())
+
+    retention_schedule_enum = ENUM(
+        'Future only',
+        '1 month',
+        '1 day',
+        '1 week',
+        '1-10 years',
+        '< 1 day',
+        '< 1 week',
+        '< 1 year',
+        '> 10 years',
+        name='retention_schedule_enum',
+        create_type=True
+    )
+    retention_schedule_enum.create(op.get_bind())
+
+    access_type_enum = ENUM(
+        'Webpage',
+        'Download',
+        'API',
+        name='access_type_enum',
+        create_type=True,
+    )
+    access_type_enum.create(op.get_bind())
+
+    for column in [
+        sa.Column('coverage_start', sa.Date(), nullable=True),
+        sa.Column('coverage_end', sa.Date(), nullable=True),
+        sa.Column("agency_supplied", sa.Boolean(), nullable=True),
+        sa.Column('agency_originated', sa.Boolean(), nullable=True),
+        sa.Column('agency_aggregation', agency_aggregation_enum),
+        sa.Column('agency_described_not_in_database', sa.Text(), nullable=True),
+        sa.Column('update_method', update_method_enum, nullable=True),
+        sa.Column('readme_url', sa.Text(), nullable=True),
+        sa.Column('originating_entity', sa.Text(), nullable=True),
+        sa.Column('retention_schedule', retention_schedule_enum, nullable=True),
+        sa.Column('scraper_url', sa.Text(), nullable=True),
+        sa.Column('submission_notes', sa.Text(), nullable=True),
+        sa.Column('access_notes', sa.Text(), nullable=True),
+        sa.Column('access_types', ARRAY(
+            access_type_enum
+        ), nullable=True),
+    ]:
+        op.add_column(
+            table_name,
+            column,
+        )
+
+def _add_autogenerated_agency_id():
+    op.execute(
+        """
+        CREATE SEQUENCE agencies_agency_id START WITH 23191;
+        """
+        )
+
+    op.execute(
+        """
+        ALTER TABLE agencies
+            ALTER COLUMN agency_id SET DEFAULT nextval('agencies_agency_id');
+               """
+    )
+
+def downgrade() -> None:
+    pass

pytest.ini

@@ -3,3 +3,4 @@ timeout = 300
 asyncio_default_fixture_loop_scope=function
 markers =
     manual: mark test as manual-only (excluded from default test runs)
+asyncio_mode = auto

src/api/endpoints/batch/dtos/post/__init__.py renamed to src/api/endpoints/agencies/__init__.py

@@ -0,0 +1,22 @@
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.db.models.impl.agency.sqlalchemy import Agency
+from src.db.queries.base.builder import QueryBuilderBase
+
+
+class DeleteAgencyQueryBuilder(QueryBuilderBase):
+
+    def __init__(
+            self,
+            agency_id: int,
+    ):
+        super().__init__()
+        self.agency_id = agency_id
+
+    async def run(self, session: AsyncSession) -> None:
+        statement = (
+            delete(Agency)
+            .where(Agency.agency_id == self.agency_id)
+        )
+        await session.execute(statement)

src/core/tasks/url/operators/probe/queries/urls/exist/__init__.py renamed to src/api/endpoints/agencies/by_id/__init__.py

@@ -0,0 +1,29 @@
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.db.models.impl.link.agency_location.sqlalchemy import LinkAgencyLocation
+from src.db.queries.base.builder import QueryBuilderBase
+
+
+class DeleteAgencyLocationQueryBuilder(QueryBuilderBase):
+
+    def __init__(
+            self,
+            agency_id: int,
+            location_id: int,
+    ):
+        super().__init__()
+        self.agency_id = agency_id
+        self.location_id = location_id
+
+    async def run(self, session: AsyncSession) -> None:
+        statement = (
+            delete(LinkAgencyLocation)
+            .where(
+                (LinkAgencyLocation.agency_id == self.agency_id)
+                & (LinkAgencyLocation.location_id == self.location_id)
+            )
+        )
+
+        await session.execute(statement)
+