From 83d458e3b41ce9b870df016c98bc0bd455aa34be Mon Sep 17 00:00:00 2001 From: Max Chis Date: Fri, 24 Oct 2025 08:26:00 -0400 Subject: [PATCH 1/2] Add linking to batch logic, remove required user id for batches --- src/api/endpoints/data_source/__init__.py | 0 src/api/endpoints/data_source/get/__init__.py | 0 src/api/endpoints/data_source/get/query.py | 0 src/api/endpoints/data_source/get/response.py | 4 +++ src/api/endpoints/data_source/put/__init__.py | 0 src/api/endpoints/data_source/put/query.py | 0 src/api/endpoints/data_source/put/request.py | 0 src/api/endpoints/data_source/routes.py | 34 +++++++++++++++++++ src/api/endpoints/meta_url/__init__.py | 0 src/api/endpoints/meta_url/get/__init__.py | 0 src/api/endpoints/meta_url/get/query.py | 0 src/api/endpoints/meta_url/get/response.py | 0 src/api/endpoints/meta_url/put/__init__.py | 0 src/api/endpoints/meta_url/put/query.py | 0 src/api/endpoints/meta_url/put/request.py | 0 src/api/endpoints/meta_url/routes.py | 34 +++++++++++++++++++ src/api/main.py | 4 ++- 17 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 src/api/endpoints/data_source/__init__.py create mode 100644 src/api/endpoints/data_source/get/__init__.py create mode 100644 src/api/endpoints/data_source/get/query.py create mode 100644 src/api/endpoints/data_source/get/response.py create mode 100644 src/api/endpoints/data_source/put/__init__.py create mode 100644 src/api/endpoints/data_source/put/query.py create mode 100644 src/api/endpoints/data_source/put/request.py create mode 100644 src/api/endpoints/data_source/routes.py create mode 100644 src/api/endpoints/meta_url/__init__.py create mode 100644 src/api/endpoints/meta_url/get/__init__.py create mode 100644 src/api/endpoints/meta_url/get/query.py create mode 100644 src/api/endpoints/meta_url/get/response.py create mode 100644 src/api/endpoints/meta_url/put/__init__.py create mode 100644 src/api/endpoints/meta_url/put/query.py create mode 100644 src/api/endpoints/meta_url/put/request.py create mode 100644 src/api/endpoints/meta_url/routes.py diff --git a/src/api/endpoints/data_source/__init__.py b/src/api/endpoints/data_source/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/data_source/get/__init__.py b/src/api/endpoints/data_source/get/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/data_source/get/query.py b/src/api/endpoints/data_source/get/query.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/data_source/get/response.py b/src/api/endpoints/data_source/get/response.py new file mode 100644 index 00000000..51134ffc --- /dev/null +++ b/src/api/endpoints/data_source/get/response.py @@ -0,0 +1,4 @@ +from pydantic import BaseModel + + +class DataSourceGetResponse(BaseModel): diff --git a/src/api/endpoints/data_source/put/__init__.py b/src/api/endpoints/data_source/put/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/data_source/put/query.py b/src/api/endpoints/data_source/put/query.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/data_source/put/request.py b/src/api/endpoints/data_source/put/request.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/data_source/routes.py b/src/api/endpoints/data_source/routes.py new file mode 100644 index 00000000..770013f6 --- /dev/null +++ b/src/api/endpoints/data_source/routes.py @@ -0,0 +1,34 @@ +from fastapi import APIRouter, Depends, Query + +from src.api.dependencies import get_async_core +from src.api.endpoints.data_source.get.response import DataSourceGetResponse +from src.api.shared.models.message_response import MessageResponse +from src.core.core import AsyncCore + +data_source_router = APIRouter( + prefix="/data-source", + tags=["data-source"] +) + + +@data_source_router.get("") +async def get_data_sources( + async_core: AsyncCore = Depends(get_async_core), + page: int = Query( + description="Page number", + default=1 + ), +) -> list[DataSourceGetResponse]: + return await async_core.adb_client.run_query_builder( + GetDataSourceQueryBuilder(page=page) + ) + +@data_source_router.put("/{data_source_id}") +async def update_data_source( + data_source_id: int, + async_core: AsyncCore = Depends(get_async_core), + request: DataSourceUpdateRequest, +) -> MessageResponse: + return await async_core.adb_client.run_query_builder( + UpdateDataSourceQueryBuilder(data_source_id=data_source_id, data_source_update=data_source_update) + ) diff --git a/src/api/endpoints/meta_url/__init__.py b/src/api/endpoints/meta_url/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/get/__init__.py b/src/api/endpoints/meta_url/get/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/get/query.py b/src/api/endpoints/meta_url/get/query.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/get/response.py b/src/api/endpoints/meta_url/get/response.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/put/__init__.py b/src/api/endpoints/meta_url/put/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/put/query.py b/src/api/endpoints/meta_url/put/query.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/put/request.py b/src/api/endpoints/meta_url/put/request.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/routes.py b/src/api/endpoints/meta_url/routes.py new file mode 100644 index 00000000..5d3f2d76 --- /dev/null +++ b/src/api/endpoints/meta_url/routes.py @@ -0,0 +1,34 @@ +from fastapi import APIRouter, Depends, Query + +from src.api.dependencies import get_async_core +from src.api.shared.models.message_response import MessageResponse +from src.core.core import AsyncCore + +meta_url_router = APIRouter( + prefix="/meta-url", + tags=["meta-url"] +) + + +@meta_url_router.get("") +async def get_meta_urls( + async_core: AsyncCore = Depends(get_async_core), + page: int = Query( + description="Page number", + default=1 + ), +) -> MetaURLGetResponse: + return await async_core.adb_client.run_query_builder(GetMetaURLQueryBuilder()) + + +@meta_url_router.put("/{meta_url_id}") +async def update_meta_url( + meta_url_id: int, + async_core: AsyncCore = Depends(get_async_core), + request: MetaURLUpdateRequest, +) -> MessageResponse: + return await async_core.adb_client.run_query_builder( + UpdateMetaURLQueryBuilder(meta_url_id=meta_url_id, meta_url_update=meta_url_update) + ) + + diff --git a/src/api/main.py b/src/api/main.py index 0026fda3..076b8108 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -12,6 +12,7 @@ from src.api.endpoints.batch.routes import batch_router from src.api.endpoints.collector.routes import collector_router from src.api.endpoints.contributions.routes import contributions_router +from src.api.endpoints.data_source.routes import data_source_router from src.api.endpoints.metrics.routes import metrics_router from src.api.endpoints.root import root_router from src.api.endpoints.search.routes import search_router @@ -179,7 +180,8 @@ async def redirect_docs(): metrics_router, submit_router, contributions_router, - agencies_router + agencies_router, + data_source_router ] for router in routers: From fcd183d06c5ed6b283a879c97793d4ea521c9846 Mon Sep 17 00:00:00 2001 From: Max Chis Date: Sat, 25 Oct 2025 15:23:42 -0400 Subject: [PATCH 2/2] Add meta url and data source endpoints --- .../endpoints/agencies/root/get/response.py | 7 +- .../data_source/{put => by_id}/__init__.py | 0 .../by_id/agency}/__init__.py | 0 .../agency/delete/__init__.py} | 0 .../by_id/agency/delete/wrapper.py | 17 ++ .../agency/get/__init__.py} | 0 .../data_source/by_id/agency/get/wrapper.py | 14 ++ .../by_id/agency/post/__init__.py} | 0 .../data_source/by_id/agency/post/wrapper.py | 17 ++ .../by_id/agency/shared/__init__.py} | 0 .../data_source/by_id/agency/shared/check.py | 17 ++ .../data_source/by_id/put/__init__.py | 0 .../endpoints/data_source/by_id/put/query.py | 123 +++++++++++++ .../data_source/by_id/put/request.py | 59 ++++++ src/api/endpoints/data_source/get/query.py | 155 ++++++++++++++++ src/api/endpoints/data_source/get/response.py | 39 ++++ src/api/endpoints/data_source/routes.py | 69 ++++++- src/api/endpoints/meta_url/by_id/__init__.py | 0 .../meta_url/by_id/agencies/__init__.py | 0 .../by_id/agencies/delete/__init__.py | 0 .../meta_url/by_id/agencies/delete/wrapper.py | 17 ++ .../meta_url/by_id/agencies/get/__init__.py | 0 .../meta_url/by_id/agencies/get/wrapper.py | 14 ++ .../meta_url/by_id/agencies/put/__init__.py | 0 .../meta_url/by_id/agencies/put/query.py | 47 +++++ .../meta_url/by_id/agencies/put/request.py | 10 + .../by_id/agencies/shared/__init__.py | 0 .../meta_url/by_id/agencies/shared/check.py | 17 ++ .../endpoints/meta_url/by_id/post/__init__.py | 0 .../endpoints/meta_url/by_id/post/wrapper.py | 17 ++ src/api/endpoints/meta_url/get/query.py | 83 +++++++++ src/api/endpoints/meta_url/get/response.py | 17 ++ src/api/endpoints/meta_url/routes.py | 72 ++++++-- src/api/main.py | 6 +- src/api/shared/agency/README.md | 1 + src/api/shared/agency/__init__.py | 0 src/api/shared/agency/delete/__init__.py | 0 src/api/shared/agency/delete/query.py | 29 +++ src/api/shared/agency/get/__init__.py | 0 src/api/shared/agency/get/query.py | 62 +++++++ src/api/shared/agency/get/response.py | 0 src/api/shared/agency/post/__init__.py | 0 src/api/shared/agency/post/query.py | 32 ++++ src/api/shared/batch/__init__.py | 0 src/api/shared/batch/url/__init__.py | 0 src/api/shared/batch/url/link.py | 36 ++++ src/api/shared/check/__init__.py | 0 src/api/shared/check/url_type/__init__.py | 0 src/api/shared/check/url_type/query.py | 58 ++++++ src/api/shared/record_type/__init__.py | 0 src/api/shared/record_type/put/__init__.py | 0 src/api/shared/record_type/put/query.py | 32 ++++ src/api/shared/url/__init__.py | 0 src/api/shared/url/put/__init__.py | 0 src/api/shared/url/put/query.py | 50 +++++ src/db/models/impl/url/core/sqlalchemy.py | 2 +- .../integration/api/data_sources/__init__.py | 0 .../api/data_sources/agencies/__init__.py | 0 .../data_sources/agencies/test_add_remove.py | 26 +++ .../agencies/test_invalid_type.py | 18 ++ .../api/data_sources/test_invalid_type.py | 20 ++ .../integration/api/data_sources/test_put.py | 89 +++++++++ .../integration/api/meta_urls/__init__.py | 0 .../api/meta_urls/agencies/__init__.py | 0 .../api/meta_urls/agencies/test_add_remove.py | 30 +++ .../meta_urls/agencies/test_invalid_type.py | 18 ++ .../api/meta_urls/test_invalid_type.py | 20 ++ .../integration/api/meta_urls/test_put.py | 39 ++++ tests/automated/integration/conftest.py | 27 ++- .../api/agencies/get/test_locations.py | 2 +- .../readonly/api/agencies/get/test_root.py | 2 +- .../readonly/api/data_sources/__init__.py | 0 .../api/data_sources/agencies/__init__.py | 0 .../api/data_sources/agencies/test_forbid.py | 13 ++ .../readonly/api/data_sources/test_get.py | 57 ++++++ .../readonly/api/meta_urls/__init__.py | 0 .../api/meta_urls/agencies/__init__.py | 0 .../api/meta_urls/agencies/test_forbid.py | 15 ++ .../readonly/api/meta_urls/test_get.py | 30 +++ .../integration/readonly/conftest.py | 61 +------ .../automated/integration/readonly/helper.py | 18 ++ tests/automated/integration/readonly/setup.py | 171 ++++++++++++++++++ .../test_submit_approved_url_task.py | 1 - tests/helpers/awaitable_barrier.py | 13 -- tests/helpers/check.py | 20 ++ tests/helpers/patch_functions.py | 10 - 86 files changed, 1709 insertions(+), 110 deletions(-) rename src/api/endpoints/data_source/{put => by_id}/__init__.py (100%) rename src/api/endpoints/{meta_url/put => data_source/by_id/agency}/__init__.py (100%) rename src/api/endpoints/data_source/{put/query.py => by_id/agency/delete/__init__.py} (100%) create mode 100644 src/api/endpoints/data_source/by_id/agency/delete/wrapper.py rename src/api/endpoints/data_source/{put/request.py => by_id/agency/get/__init__.py} (100%) create mode 100644 src/api/endpoints/data_source/by_id/agency/get/wrapper.py rename src/api/endpoints/{meta_url/put/query.py => data_source/by_id/agency/post/__init__.py} (100%) create mode 100644 src/api/endpoints/data_source/by_id/agency/post/wrapper.py rename src/api/endpoints/{meta_url/put/request.py => data_source/by_id/agency/shared/__init__.py} (100%) create mode 100644 src/api/endpoints/data_source/by_id/agency/shared/check.py create mode 100644 src/api/endpoints/data_source/by_id/put/__init__.py create mode 100644 src/api/endpoints/data_source/by_id/put/query.py create mode 100644 src/api/endpoints/data_source/by_id/put/request.py create mode 100644 src/api/endpoints/meta_url/by_id/__init__.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/__init__.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/delete/__init__.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/delete/wrapper.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/get/__init__.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/get/wrapper.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/put/__init__.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/put/query.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/put/request.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/shared/__init__.py create mode 100644 src/api/endpoints/meta_url/by_id/agencies/shared/check.py create mode 100644 src/api/endpoints/meta_url/by_id/post/__init__.py create mode 100644 src/api/endpoints/meta_url/by_id/post/wrapper.py create mode 100644 src/api/shared/agency/README.md create mode 100644 src/api/shared/agency/__init__.py create mode 100644 src/api/shared/agency/delete/__init__.py create mode 100644 src/api/shared/agency/delete/query.py create mode 100644 src/api/shared/agency/get/__init__.py create mode 100644 src/api/shared/agency/get/query.py create mode 100644 src/api/shared/agency/get/response.py create mode 100644 src/api/shared/agency/post/__init__.py create mode 100644 src/api/shared/agency/post/query.py create mode 100644 src/api/shared/batch/__init__.py create mode 100644 src/api/shared/batch/url/__init__.py create mode 100644 src/api/shared/batch/url/link.py create mode 100644 src/api/shared/check/__init__.py create mode 100644 src/api/shared/check/url_type/__init__.py create mode 100644 src/api/shared/check/url_type/query.py create mode 100644 src/api/shared/record_type/__init__.py create mode 100644 src/api/shared/record_type/put/__init__.py create mode 100644 src/api/shared/record_type/put/query.py create mode 100644 src/api/shared/url/__init__.py create mode 100644 src/api/shared/url/put/__init__.py create mode 100644 src/api/shared/url/put/query.py create mode 100644 tests/automated/integration/api/data_sources/__init__.py create mode 100644 tests/automated/integration/api/data_sources/agencies/__init__.py create mode 100644 tests/automated/integration/api/data_sources/agencies/test_add_remove.py create mode 100644 tests/automated/integration/api/data_sources/agencies/test_invalid_type.py create mode 100644 tests/automated/integration/api/data_sources/test_invalid_type.py create mode 100644 tests/automated/integration/api/data_sources/test_put.py create mode 100644 tests/automated/integration/api/meta_urls/__init__.py create mode 100644 tests/automated/integration/api/meta_urls/agencies/__init__.py create mode 100644 tests/automated/integration/api/meta_urls/agencies/test_add_remove.py create mode 100644 tests/automated/integration/api/meta_urls/agencies/test_invalid_type.py create mode 100644 tests/automated/integration/api/meta_urls/test_invalid_type.py create mode 100644 tests/automated/integration/api/meta_urls/test_put.py create mode 100644 tests/automated/integration/readonly/api/data_sources/__init__.py create mode 100644 tests/automated/integration/readonly/api/data_sources/agencies/__init__.py create mode 100644 tests/automated/integration/readonly/api/data_sources/agencies/test_forbid.py create mode 100644 tests/automated/integration/readonly/api/data_sources/test_get.py create mode 100644 tests/automated/integration/readonly/api/meta_urls/__init__.py create mode 100644 tests/automated/integration/readonly/api/meta_urls/agencies/__init__.py create mode 100644 tests/automated/integration/readonly/api/meta_urls/agencies/test_forbid.py create mode 100644 tests/automated/integration/readonly/api/meta_urls/test_get.py create mode 100644 tests/automated/integration/readonly/helper.py create mode 100644 tests/automated/integration/readonly/setup.py delete mode 100644 tests/helpers/awaitable_barrier.py create mode 100644 tests/helpers/check.py delete mode 100644 tests/helpers/patch_functions.py diff --git a/src/api/endpoints/agencies/root/get/response.py b/src/api/endpoints/agencies/root/get/response.py index b9d374eb..23590958 100644 --- a/src/api/endpoints/agencies/root/get/response.py +++ b/src/api/endpoints/agencies/root/get/response.py @@ -8,5 +8,8 @@ class AgencyGetResponse(BaseModel): id: int name: str type: AgencyType - jurisdiction_type: JurisdictionType - locations: list[AgencyGetLocationsResponse] \ No newline at end of file + jurisdiction_type: JurisdictionType | None + locations: list[AgencyGetLocationsResponse] + +class AgencyGetOuterResponse(BaseModel): + results: list[AgencyGetResponse] \ No newline at end of file diff --git a/src/api/endpoints/data_source/put/__init__.py b/src/api/endpoints/data_source/by_id/__init__.py similarity index 100% rename from src/api/endpoints/data_source/put/__init__.py rename to src/api/endpoints/data_source/by_id/__init__.py diff --git a/src/api/endpoints/meta_url/put/__init__.py b/src/api/endpoints/data_source/by_id/agency/__init__.py similarity index 100% rename from src/api/endpoints/meta_url/put/__init__.py rename to src/api/endpoints/data_source/by_id/agency/__init__.py diff --git a/src/api/endpoints/data_source/put/query.py b/src/api/endpoints/data_source/by_id/agency/delete/__init__.py similarity index 100% rename from src/api/endpoints/data_source/put/query.py rename to src/api/endpoints/data_source/by_id/agency/delete/__init__.py diff --git a/src/api/endpoints/data_source/by_id/agency/delete/wrapper.py b/src/api/endpoints/data_source/by_id/agency/delete/wrapper.py new file mode 100644 index 00000000..f04885af --- /dev/null +++ b/src/api/endpoints/data_source/by_id/agency/delete/wrapper.py @@ -0,0 +1,17 @@ +from src.api.endpoints.data_source.by_id.agency.shared.check import check_is_data_source_url +from src.api.shared.agency.delete.query import RemoveURLAgencyLinkQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient + + +async def delete_data_source_agency_link( + url_id: int, + agency_id: int, + adb_client: AsyncDatabaseClient +) -> None: + await check_is_data_source_url(url_id=url_id, adb_client=adb_client) + await adb_client.run_query_builder( + RemoveURLAgencyLinkQueryBuilder( + url_id=url_id, + agency_id=agency_id + ) + ) \ No newline at end of file diff --git a/src/api/endpoints/data_source/put/request.py b/src/api/endpoints/data_source/by_id/agency/get/__init__.py similarity index 100% rename from src/api/endpoints/data_source/put/request.py rename to src/api/endpoints/data_source/by_id/agency/get/__init__.py diff --git a/src/api/endpoints/data_source/by_id/agency/get/wrapper.py b/src/api/endpoints/data_source/by_id/agency/get/wrapper.py new file mode 100644 index 00000000..f58d4936 --- /dev/null +++ b/src/api/endpoints/data_source/by_id/agency/get/wrapper.py @@ -0,0 +1,14 @@ +from src.api.endpoints.agencies.root.get.response import AgencyGetOuterResponse +from src.api.endpoints.data_source.by_id.agency.shared.check import check_is_data_source_url +from src.api.shared.agency.get.query import GetRelatedAgenciesQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient + + +async def get_data_source_agencies_wrapper( + url_id: int, + adb_client: AsyncDatabaseClient +) -> AgencyGetOuterResponse: + await check_is_data_source_url(url_id=url_id, adb_client=adb_client) + return await adb_client.run_query_builder( + GetRelatedAgenciesQueryBuilder(url_id=url_id) + ) \ No newline at end of file diff --git a/src/api/endpoints/meta_url/put/query.py b/src/api/endpoints/data_source/by_id/agency/post/__init__.py similarity index 100% rename from src/api/endpoints/meta_url/put/query.py rename to src/api/endpoints/data_source/by_id/agency/post/__init__.py diff --git a/src/api/endpoints/data_source/by_id/agency/post/wrapper.py b/src/api/endpoints/data_source/by_id/agency/post/wrapper.py new file mode 100644 index 00000000..97197103 --- /dev/null +++ b/src/api/endpoints/data_source/by_id/agency/post/wrapper.py @@ -0,0 +1,17 @@ +from src.api.endpoints.data_source.by_id.agency.shared.check import check_is_data_source_url +from src.api.shared.agency.post.query import AddURLAgencyLinkQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient + + +async def add_data_source_agency_link( + url_id: int, + agency_id: int, + adb_client: AsyncDatabaseClient +) -> None: + await check_is_data_source_url(url_id=url_id, adb_client=adb_client) + await adb_client.run_query_builder( + AddURLAgencyLinkQueryBuilder( + url_id=url_id, + agency_id=agency_id + ) + ) \ No newline at end of file diff --git a/src/api/endpoints/meta_url/put/request.py b/src/api/endpoints/data_source/by_id/agency/shared/__init__.py similarity index 100% rename from src/api/endpoints/meta_url/put/request.py rename to src/api/endpoints/data_source/by_id/agency/shared/__init__.py diff --git a/src/api/endpoints/data_source/by_id/agency/shared/check.py b/src/api/endpoints/data_source/by_id/agency/shared/check.py new file mode 100644 index 00000000..2ef9640c --- /dev/null +++ b/src/api/endpoints/data_source/by_id/agency/shared/check.py @@ -0,0 +1,17 @@ +from src.api.shared.check.url_type.query import CheckURLTypeQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.flag.url_validated.enums import URLType + + +async def check_is_data_source_url( + url_id: int, + adb_client: AsyncDatabaseClient +) -> None: + """ + Raises: + Bad Request if url_type is not valid or does not exist + """ + + await adb_client.run_query_builder( + CheckURLTypeQueryBuilder(url_id=url_id, url_type=URLType.DATA_SOURCE) + ) \ No newline at end of file diff --git a/src/api/endpoints/data_source/by_id/put/__init__.py b/src/api/endpoints/data_source/by_id/put/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/data_source/by_id/put/query.py b/src/api/endpoints/data_source/by_id/put/query.py new file mode 100644 index 00000000..96106395 --- /dev/null +++ b/src/api/endpoints/data_source/by_id/put/query.py @@ -0,0 +1,123 @@ +from sqlalchemy import update, select, literal, insert +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.endpoints.data_source.by_id.put.request import DataSourcePutRequest +from src.api.shared.batch.url.link import UpdateBatchURLLinkQueryBuilder +from src.api.shared.record_type.put.query import UpdateRecordTypeQueryBuilder +from src.api.shared.url.put.query import UpdateURLQueryBuilder +from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata +from src.db.queries.base.builder import QueryBuilderBase + + +class UpdateDataSourceQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int, + request: DataSourcePutRequest, + ): + super().__init__() + self.url_id = url_id + self.request = request + + async def run(self, session: AsyncSession) -> None: + + if self.request.record_type is not None: + await UpdateRecordTypeQueryBuilder( + url_id=self.url_id, + record_type=self.request.record_type + ).run(session) + + # Update URL if any of the URL fields are not None + if ( + self.request.description is None and + self.request.name is None and + self.request.description is None + ): + return + + # Update Batch if Batch link is none + if self.request.batch_id is not None: + await UpdateBatchURLLinkQueryBuilder( + batch_id=self.request.batch_id, + url_id=self.url_id + ).run(session) + + await UpdateURLQueryBuilder( + url_id=self.url_id, + url=self.request.url, + name=self.request.name, + description=self.request.description, + ).run( + session, + ) + if not self.request.optional_data_source_metadata_not_none(): + return + value_dict = {} + if self.request.record_formats is not None: + value_dict["record_formats"] = self.request.record_formats + if self.request.data_portal_type is not None: + value_dict["data_portal_type"] = self.request.data_portal_type + if self.request.supplying_entity is not None: + value_dict["supplying_entity"] = self.request.supplying_entity + if self.request.coverage_start is not None: + value_dict["coverage_start"] = self.request.coverage_start + if self.request.coverage_end is not None: + value_dict["coverage_end"] = self.request.coverage_end + if self.request.agency_supplied is not None: + value_dict["agency_supplied"] = self.request.agency_supplied + if self.request.agency_originated is not None: + value_dict["agency_originated"] = self.request.agency_originated + if self.request.agency_aggregation is not None: + value_dict["agency_aggregation"] = self.request.agency_aggregation + if self.request.agency_described_not_in_database is not None: + value_dict["agency_described_not_in_database"] = self.request.agency_described_not_in_database + if self.request.update_method is not None: + value_dict["update_method"] = self.request.update_method + if self.request.readme_url is not None: + value_dict["readme_url"] = self.request.readme_url + if self.request.originating_entity is not None: + value_dict["originating_entity"] = self.request.originating_entity + if self.request.retention_schedule is not None: + value_dict["retention_schedule"] = self.request.retention_schedule + if self.request.scraper_url is not None: + value_dict["scraper_url"] = self.request.scraper_url + if self.request.submission_notes is not None: + value_dict["submission_notes"] = self.request.submission_notes + if self.request.access_notes is not None: + value_dict["access_notes"] = self.request.access_notes + if self.request.access_types is not None: + value_dict["access_types"] = self.request.access_types + + # Check for existing metadata object + query = ( + select( + literal(True) + ) + .where( + URLOptionalDataSourceMetadata.url_id == self.url_id + ) + ) + exists = await self.sh.one_or_none(session=session, query=query) + if not exists: + insert_obj = URLOptionalDataSourceMetadata( + url_id=self.url_id, + **value_dict + ) + session.add(insert_obj) + else: + statement = ( + update( + URLOptionalDataSourceMetadata + ) + .where( + URLOptionalDataSourceMetadata.url_id == self.url_id + ) + .values( + value_dict + ) + ) + + await session.execute(statement) + + diff --git a/src/api/endpoints/data_source/by_id/put/request.py b/src/api/endpoints/data_source/by_id/put/request.py new file mode 100644 index 00000000..28549c28 --- /dev/null +++ b/src/api/endpoints/data_source/by_id/put/request.py @@ -0,0 +1,59 @@ +from datetime import date + +from pydantic import BaseModel + +from src.core.enums import RecordType +from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \ + RetentionScheduleEnum, AccessTypeEnum + + +class DataSourcePutRequest(BaseModel): + + # Required Attributes + url: str | None = None + name: str | None = None + record_type: RecordType | None = None + + # Optional Attributes + batch_id: int | None = None + description: str | None = None + + # Optional data source metadata + record_formats: list[str] | None = None + data_portal_type: str | None = None + supplying_entity: str | None = None + coverage_start: date | None = None + coverage_end: date | None = None + agency_supplied: bool | None = None + agency_originated: bool | None = None + agency_aggregation: AgencyAggregationEnum | None = None + agency_described_not_in_database: str | None = None + update_method: UpdateMethodEnum | None = None + readme_url: str | None = None + originating_entity: str | None = None + retention_schedule: RetentionScheduleEnum | None = None + scraper_url: str | None = None + submission_notes: str | None = None + access_notes: str | None = None + access_types: list[AccessTypeEnum] | None = None + + def optional_data_source_metadata_not_none(self) -> bool: + return ( + self.record_formats is not None or + self.data_portal_type is not None or + self.supplying_entity is not None or + self.coverage_start is not None or + self.coverage_end is not None or + self.agency_supplied is not None or + self.agency_originated is not None or + self.agency_aggregation is not None or + self.agency_described_not_in_database is not None or + self.update_method is not None or + self.readme_url is not None or + self.originating_entity is not None or + self.retention_schedule is not None or + self.scraper_url is not None or + self.submission_notes is not None or + self.access_notes is not None or + self.access_types is not None + ) \ No newline at end of file diff --git a/src/api/endpoints/data_source/get/query.py b/src/api/endpoints/data_source/get/query.py index e69de29b..e9d0598b 100644 --- a/src/api/endpoints/data_source/get/query.py +++ b/src/api/endpoints/data_source/get/query.py @@ -0,0 +1,155 @@ +from datetime import date +from typing import Any, Sequence + +from sqlalchemy import select, RowMapping, and_ +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from src.api.endpoints.data_source.get.response import DataSourceGetOuterResponse, DataSourceGetResponse +from src.core.enums import RecordType +from src.db.models.impl.flag.url_validated.enums import URLType +from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated +from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \ + RetentionScheduleEnum, AccessTypeEnum +from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata +from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType +from src.db.queries.base.builder import QueryBuilderBase + + +class GetDataSourceQueryBuilder(QueryBuilderBase): + + def __init__( + self, + page: int, + ): + super().__init__() + self.page = page + + async def run(self, session: AsyncSession) -> DataSourceGetOuterResponse: + query = ( + select( + URL, + URL.id, + URL.url, + + # Required Attributes + URL.name, + URLRecordType.record_type, + URL.confirmed_agencies, + + # Optional Attributes + URL.description, + LinkBatchURL.batch_id, + URLOptionalDataSourceMetadata.record_formats, + URLOptionalDataSourceMetadata.data_portal_type, + URLOptionalDataSourceMetadata.supplying_entity, + URLOptionalDataSourceMetadata.coverage_start, + URLOptionalDataSourceMetadata.coverage_end, + URLOptionalDataSourceMetadata.agency_supplied, + URLOptionalDataSourceMetadata.agency_aggregation, + URLOptionalDataSourceMetadata.agency_described_not_in_database, + URLOptionalDataSourceMetadata.agency_originated, + URLOptionalDataSourceMetadata.update_method, + URLOptionalDataSourceMetadata.readme_url, + URLOptionalDataSourceMetadata.originating_entity, + URLOptionalDataSourceMetadata.retention_schedule, + URLOptionalDataSourceMetadata.scraper_url, + URLOptionalDataSourceMetadata.submission_notes, + URLOptionalDataSourceMetadata.access_notes, + URLOptionalDataSourceMetadata.access_types + ) + .join( + URLRecordType, + URLRecordType.url_id == URL.id + ) + .join( + FlagURLValidated, + and_( + FlagURLValidated.url_id == URL.id, + FlagURLValidated.type == URLType.DATA_SOURCE + ) + ) + .outerjoin( + LinkBatchURL, + LinkBatchURL.url_id == URL.id + ) + .outerjoin( + URLOptionalDataSourceMetadata, + URLOptionalDataSourceMetadata.url_id == URL.id + ) + .options( + selectinload(URL.confirmed_agencies), + ) + .limit(100) + .offset((self.page - 1) * 100) + ) + + mappings: Sequence[RowMapping] = await self.sh.mappings(session, query=query) + responses: list[DataSourceGetResponse] = [] + + for mapping in mappings: + url: URL = mapping[URL] + url_id: int = mapping[URL.id] + url_url: str = mapping[URL.url] + url_name: str = mapping[URL.name] + url_record_type: RecordType = mapping[URLRecordType.record_type] + + url_agency_ids: list[int] = [] + for agency in url.confirmed_agencies: + url_agency_ids.append(agency.agency_id) + + url_description: str | None = mapping[URL.description] + link_batch_url_batch_id: int | None = mapping[LinkBatchURL.batch_id] + url_record_formats: list[str] | None = mapping[URLOptionalDataSourceMetadata.record_formats] + url_data_portal_type: str | None = mapping[URLOptionalDataSourceMetadata.data_portal_type] + url_supplying_entity: str | None = mapping[URLOptionalDataSourceMetadata.supplying_entity] + url_coverage_start: date | None = mapping[URLOptionalDataSourceMetadata.coverage_start] + url_coverage_end: date | None = mapping[URLOptionalDataSourceMetadata.coverage_end] + url_agency_supplied: bool | None = mapping[URLOptionalDataSourceMetadata.agency_supplied] + url_agency_aggregation: AgencyAggregationEnum | None = mapping[URLOptionalDataSourceMetadata.agency_aggregation] + url_agency_originated: bool | None = mapping[URLOptionalDataSourceMetadata.agency_originated] + url_agency_described_not_in_database: bool | None = mapping[URLOptionalDataSourceMetadata.agency_described_not_in_database] + url_update_method: UpdateMethodEnum | None = mapping[URLOptionalDataSourceMetadata.update_method] + url_readme_url: str | None = mapping[URLOptionalDataSourceMetadata.readme_url] + url_originating_entity: str | None = mapping[URLOptionalDataSourceMetadata.originating_entity] + url_retention_schedule: RetentionScheduleEnum | None = mapping[URLOptionalDataSourceMetadata.retention_schedule] + url_scraper_url: str | None = mapping[URLOptionalDataSourceMetadata.scraper_url] + url_submission_notes: str | None = mapping[URLOptionalDataSourceMetadata.submission_notes] + url_access_notes: str | None = mapping[URLOptionalDataSourceMetadata.access_notes] + url_access_types: list[AccessTypeEnum] | None = mapping[URLOptionalDataSourceMetadata.access_types] + + responses.append( + DataSourceGetResponse( + url_id=url_id, + url=url_url, + name=url_name, + record_type=url_record_type, + agency_ids=url_agency_ids, + description=url_description, + batch_id=link_batch_url_batch_id, + record_formats=url_record_formats, + data_portal_type=url_data_portal_type, + supplying_entity=url_supplying_entity, + coverage_start=url_coverage_start, + coverage_end=url_coverage_end, + agency_supplied=url_agency_supplied, + agency_aggregation=url_agency_aggregation, + agency_originated=url_agency_originated, + agency_described_not_in_database=url_agency_described_not_in_database, + update_method=url_update_method, + readme_url=url_readme_url, + originating_entity=url_originating_entity, + retention_schedule=url_retention_schedule, + scraper_url=url_scraper_url, + submission_notes=url_submission_notes, + access_notes=url_access_notes, + access_types=url_access_types + ) + ) + + return DataSourceGetOuterResponse( + results=responses, + ) + diff --git a/src/api/endpoints/data_source/get/response.py b/src/api/endpoints/data_source/get/response.py index 51134ffc..b80ee9e1 100644 --- a/src/api/endpoints/data_source/get/response.py +++ b/src/api/endpoints/data_source/get/response.py @@ -1,4 +1,43 @@ +from datetime import date + from pydantic import BaseModel +from src.core.enums import RecordType +from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \ + RetentionScheduleEnum, AccessTypeEnum + class DataSourceGetResponse(BaseModel): + url_id: int + url: str + + # Required Attributes + name: str + record_type: RecordType + agency_ids: list[int] + + # Optional Attributes + batch_id: int | None + description: str | None + + # Optional data source metadata + record_formats: list[str] + data_portal_type: str | None = None + supplying_entity: str | None = None + coverage_start: date | None = None + coverage_end: date | None = None + agency_supplied: bool | None = None + agency_originated: bool | None = None + agency_aggregation: AgencyAggregationEnum | None = None + agency_described_not_in_database: str | None = None + update_method: UpdateMethodEnum | None = None + readme_url: str | None = None + originating_entity: str | None = None + retention_schedule: RetentionScheduleEnum | None = None + scraper_url: str | None = None + submission_notes: str | None = None + access_notes: str | None = None + access_types: list[AccessTypeEnum] + +class DataSourceGetOuterResponse(BaseModel): + results: list[DataSourceGetResponse] diff --git a/src/api/endpoints/data_source/routes.py b/src/api/endpoints/data_source/routes.py index 770013f6..2464ceea 100644 --- a/src/api/endpoints/data_source/routes.py +++ b/src/api/endpoints/data_source/routes.py @@ -1,34 +1,83 @@ from fastapi import APIRouter, Depends, Query from src.api.dependencies import get_async_core -from src.api.endpoints.data_source.get.response import DataSourceGetResponse +from src.api.endpoints.agencies.root.get.response import AgencyGetResponse, AgencyGetOuterResponse +from src.api.endpoints.data_source.by_id.agency.delete.wrapper import delete_data_source_agency_link +from src.api.endpoints.data_source.by_id.agency.get.wrapper import get_data_source_agencies_wrapper +from src.api.endpoints.data_source.by_id.agency.post.wrapper import add_data_source_agency_link +from src.api.endpoints.data_source.by_id.agency.shared.check import check_is_data_source_url +from src.api.endpoints.data_source.get.query import GetDataSourceQueryBuilder +from src.api.endpoints.data_source.get.response import DataSourceGetOuterResponse +from src.api.endpoints.data_source.by_id.put.query import UpdateDataSourceQueryBuilder +from src.api.endpoints.data_source.by_id.put.request import DataSourcePutRequest from src.api.shared.models.message_response import MessageResponse from src.core.core import AsyncCore -data_source_router = APIRouter( - prefix="/data-source", +data_sources_router = APIRouter( + prefix="/data-sources", tags=["data-source"] ) -@data_source_router.get("") +@data_sources_router.get("") async def get_data_sources( async_core: AsyncCore = Depends(get_async_core), page: int = Query( description="Page number", default=1 ), -) -> list[DataSourceGetResponse]: +) -> DataSourceGetOuterResponse: return await async_core.adb_client.run_query_builder( GetDataSourceQueryBuilder(page=page) ) -@data_source_router.put("/{data_source_id}") +@data_sources_router.put("/{url_id}") async def update_data_source( - data_source_id: int, + url_id: int , + request: DataSourcePutRequest, async_core: AsyncCore = Depends(get_async_core), - request: DataSourceUpdateRequest, ) -> MessageResponse: - return await async_core.adb_client.run_query_builder( - UpdateDataSourceQueryBuilder(data_source_id=data_source_id, data_source_update=data_source_update) + await check_is_data_source_url(url_id=url_id, adb_client=async_core.adb_client) + await async_core.adb_client.run_query_builder( + UpdateDataSourceQueryBuilder( + url_id=url_id, + request=request + ) + ) + return MessageResponse(message="Data source updated.") + +@data_sources_router.get("/{url_id}/agencies") +async def get_data_source_agencies( + url_id: int, + async_core: AsyncCore = Depends(get_async_core), +) -> AgencyGetOuterResponse: + return await get_data_source_agencies_wrapper( + url_id=url_id, + adb_client=async_core.adb_client + ) + +@data_sources_router.post("/{url_id}/agencies/{agency_id}") +async def add_agency_to_data_source( + url_id: int, + agency_id: int, + async_core: AsyncCore = Depends(get_async_core), +) -> MessageResponse: + await add_data_source_agency_link( + url_id=url_id, + agency_id=agency_id, + adb_client=async_core.adb_client + ) + return MessageResponse(message="Agency added to data source.") + +@data_sources_router.delete("/{url_id}/agencies/{agency_id}") +async def remove_agency_from_data_source( + url_id: int, + agency_id: int, + async_core: AsyncCore = Depends(get_async_core), +) -> MessageResponse: + await delete_data_source_agency_link( + url_id=url_id, + agency_id=agency_id, + adb_client=async_core.adb_client ) + return MessageResponse(message="Agency removed from data source.") diff --git a/src/api/endpoints/meta_url/by_id/__init__.py b/src/api/endpoints/meta_url/by_id/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/by_id/agencies/__init__.py b/src/api/endpoints/meta_url/by_id/agencies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/by_id/agencies/delete/__init__.py b/src/api/endpoints/meta_url/by_id/agencies/delete/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/by_id/agencies/delete/wrapper.py b/src/api/endpoints/meta_url/by_id/agencies/delete/wrapper.py new file mode 100644 index 00000000..7adf695a --- /dev/null +++ b/src/api/endpoints/meta_url/by_id/agencies/delete/wrapper.py @@ -0,0 +1,17 @@ +from src.api.endpoints.meta_url.by_id.agencies.shared.check import check_is_meta_url +from src.api.shared.agency.delete.query import RemoveURLAgencyLinkQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient + + +async def delete_meta_url_agency_link( + url_id: int, + agency_id: int, + adb_client: AsyncDatabaseClient +) -> None: + await check_is_meta_url(url_id=url_id, adb_client=adb_client) + await adb_client.run_query_builder( + RemoveURLAgencyLinkQueryBuilder( + url_id=url_id, + agency_id=agency_id + ) + ) \ No newline at end of file diff --git a/src/api/endpoints/meta_url/by_id/agencies/get/__init__.py b/src/api/endpoints/meta_url/by_id/agencies/get/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/by_id/agencies/get/wrapper.py b/src/api/endpoints/meta_url/by_id/agencies/get/wrapper.py new file mode 100644 index 00000000..17362a88 --- /dev/null +++ b/src/api/endpoints/meta_url/by_id/agencies/get/wrapper.py @@ -0,0 +1,14 @@ +from src.api.endpoints.agencies.root.get.response import AgencyGetOuterResponse +from src.api.endpoints.meta_url.by_id.agencies.shared.check import check_is_meta_url +from src.api.shared.agency.get.query import GetRelatedAgenciesQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient + + +async def get_meta_url_agencies_wrapper( + url_id: int, + adb_client: AsyncDatabaseClient +) -> AgencyGetOuterResponse: + await check_is_meta_url(url_id=url_id, adb_client=adb_client) + return await adb_client.run_query_builder( + GetRelatedAgenciesQueryBuilder(url_id=url_id) + ) \ No newline at end of file diff --git a/src/api/endpoints/meta_url/by_id/agencies/put/__init__.py b/src/api/endpoints/meta_url/by_id/agencies/put/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/by_id/agencies/put/query.py b/src/api/endpoints/meta_url/by_id/agencies/put/query.py new file mode 100644 index 00000000..a3be8cf8 --- /dev/null +++ b/src/api/endpoints/meta_url/by_id/agencies/put/query.py @@ -0,0 +1,47 @@ +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.endpoints.meta_url.by_id.agencies.put.request import UpdateMetaURLRequest +from src.api.shared.batch.url.link import UpdateBatchURLLinkQueryBuilder +from src.api.shared.record_type.put.query import UpdateRecordTypeQueryBuilder +from src.api.shared.url.put.query import UpdateURLQueryBuilder +from src.db.queries.base.builder import QueryBuilderBase + + +class UpdateMetaURLQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int, + request: UpdateMetaURLRequest + ): + super().__init__() + self.url_id = url_id + self.request = request + + async def run(self, session: AsyncSession) -> None: + + # Update Batch ID if not none + if self.request.batch_id is not None: + await UpdateBatchURLLinkQueryBuilder( + batch_id=self.request.batch_id, + url_id=self.url_id + ).run(session) + + + # Update URL if any of the URL fields are not None + if ( + self.request.description is None and + self.request.name is None and + self.request.description is None + ): + return + + await UpdateURLQueryBuilder( + url_id=self.url_id, + url=self.request.url, + name=self.request.name, + description=self.request.description, + ).run( + session, + ) + diff --git a/src/api/endpoints/meta_url/by_id/agencies/put/request.py b/src/api/endpoints/meta_url/by_id/agencies/put/request.py new file mode 100644 index 00000000..456f2b99 --- /dev/null +++ b/src/api/endpoints/meta_url/by_id/agencies/put/request.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + + +class UpdateMetaURLRequest(BaseModel): + url: str | None = None + name: str | None = None + description: str | None = None + + batch_id: int | None = None + diff --git a/src/api/endpoints/meta_url/by_id/agencies/shared/__init__.py b/src/api/endpoints/meta_url/by_id/agencies/shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/by_id/agencies/shared/check.py b/src/api/endpoints/meta_url/by_id/agencies/shared/check.py new file mode 100644 index 00000000..72c79601 --- /dev/null +++ b/src/api/endpoints/meta_url/by_id/agencies/shared/check.py @@ -0,0 +1,17 @@ +from src.api.shared.check.url_type.query import CheckURLTypeQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.flag.url_validated.enums import URLType + + +async def check_is_meta_url( + url_id: int, + adb_client: AsyncDatabaseClient +) -> None: + """ + Raises: + Bad Request if url_type is not valid or does not exist + """ + + await adb_client.run_query_builder( + CheckURLTypeQueryBuilder(url_id=url_id, url_type=URLType.META_URL) + ) \ No newline at end of file diff --git a/src/api/endpoints/meta_url/by_id/post/__init__.py b/src/api/endpoints/meta_url/by_id/post/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/meta_url/by_id/post/wrapper.py b/src/api/endpoints/meta_url/by_id/post/wrapper.py new file mode 100644 index 00000000..4153e144 --- /dev/null +++ b/src/api/endpoints/meta_url/by_id/post/wrapper.py @@ -0,0 +1,17 @@ +from src.api.endpoints.meta_url.by_id.agencies.shared.check import check_is_meta_url +from src.api.shared.agency.post.query import AddURLAgencyLinkQueryBuilder +from src.db.client.async_ import AsyncDatabaseClient + + +async def add_meta_url_agency_link( + url_id: int, + agency_id: int, + adb_client: AsyncDatabaseClient +) -> None: + await check_is_meta_url(url_id=url_id, adb_client=adb_client) + await adb_client.run_query_builder( + AddURLAgencyLinkQueryBuilder( + url_id=url_id, + agency_id=agency_id + ) + ) \ No newline at end of file diff --git a/src/api/endpoints/meta_url/get/query.py b/src/api/endpoints/meta_url/get/query.py index e69de29b..202626d8 100644 --- a/src/api/endpoints/meta_url/get/query.py +++ b/src/api/endpoints/meta_url/get/query.py @@ -0,0 +1,83 @@ +from typing import Sequence + +from sqlalchemy import select, and_, RowMapping +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from src.api.endpoints.meta_url.get.response import MetaURLGetOuterResponse, MetaURLGetResponse +from src.db.models.impl.flag.url_validated.enums import URLType +from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated +from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.queries.base.builder import QueryBuilderBase + + +class GetMetaURLQueryBuilder(QueryBuilderBase): + + def __init__( + self, + page: int, + ): + super().__init__() + self.page = page + + async def run(self, session: AsyncSession) -> MetaURLGetOuterResponse: + query = ( + select( + URL, + URL.id, + URL.url, + + # Required Attributes + URL.name, + URL.confirmed_agencies, + + # Optional Attributes + URL.description, + LinkBatchURL.batch_id, + ) + .join( + FlagURLValidated, + and_( + FlagURLValidated.url_id == URL.id, + FlagURLValidated.type == URLType.META_URL + ) + ) + .outerjoin( + LinkBatchURL, + LinkBatchURL.url_id == URL.id + ) + .options( + selectinload(URL.confirmed_agencies), + ) + .limit(100) + .offset((self.page - 1) * 100) + ) + + mappings: Sequence[RowMapping] = await self.sh.mappings(session, query=query) + responses: list[MetaURLGetResponse] = [] + + for mapping in mappings: + url: URL = mapping[URL] + url_id: int = mapping[URL.id] + url_url: str = mapping[URL.url] + url_name: str = mapping[URL.name] + url_agency_ids: list[int] = [] + for agency in url.confirmed_agencies: + url_agency_ids.append(agency.agency_id) + url_description: str | None = mapping[URL.description] + link_batch_url_batch_id: int | None = mapping[LinkBatchURL.batch_id] + responses.append( + MetaURLGetResponse( + url_id=url_id, + url=url_url, + name=url_name, + agency_ids=url_agency_ids, + description=url_description, + batch_id=link_batch_url_batch_id, + ) + ) + + return MetaURLGetOuterResponse( + results=responses, + ) \ No newline at end of file diff --git a/src/api/endpoints/meta_url/get/response.py b/src/api/endpoints/meta_url/get/response.py index e69de29b..1f683a65 100644 --- a/src/api/endpoints/meta_url/get/response.py +++ b/src/api/endpoints/meta_url/get/response.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel + + +class MetaURLGetResponse(BaseModel): + url_id: int + url: str + + # Required Attributes + name: str + agency_ids: list[int] + + # Optional Attributes + batch_id: int| None + description: str | None + +class MetaURLGetOuterResponse(BaseModel): + results: list[MetaURLGetResponse] \ No newline at end of file diff --git a/src/api/endpoints/meta_url/routes.py b/src/api/endpoints/meta_url/routes.py index 5d3f2d76..0f14805c 100644 --- a/src/api/endpoints/meta_url/routes.py +++ b/src/api/endpoints/meta_url/routes.py @@ -1,34 +1,84 @@ from fastapi import APIRouter, Depends, Query from src.api.dependencies import get_async_core +from src.api.endpoints.agencies.root.get.response import AgencyGetResponse, AgencyGetOuterResponse +from src.api.endpoints.meta_url.by_id.agencies.delete.wrapper import delete_meta_url_agency_link +from src.api.endpoints.meta_url.by_id.agencies.get.wrapper import get_meta_url_agencies_wrapper +from src.api.endpoints.meta_url.by_id.agencies.shared.check import check_is_meta_url +from src.api.endpoints.meta_url.by_id.post.wrapper import add_meta_url_agency_link +from src.api.endpoints.meta_url.get.query import GetMetaURLQueryBuilder +from src.api.endpoints.meta_url.get.response import MetaURLGetResponse, MetaURLGetOuterResponse +from src.api.endpoints.meta_url.by_id.agencies.put.query import UpdateMetaURLQueryBuilder +from src.api.endpoints.meta_url.by_id.agencies.put.request import UpdateMetaURLRequest from src.api.shared.models.message_response import MessageResponse from src.core.core import AsyncCore -meta_url_router = APIRouter( - prefix="/meta-url", +meta_urls_router = APIRouter( + prefix="/meta-urls", tags=["meta-url"] ) - -@meta_url_router.get("") +@meta_urls_router.get("") async def get_meta_urls( async_core: AsyncCore = Depends(get_async_core), page: int = Query( description="Page number", default=1 ), -) -> MetaURLGetResponse: - return await async_core.adb_client.run_query_builder(GetMetaURLQueryBuilder()) +) -> MetaURLGetOuterResponse: + return await async_core.adb_client.run_query_builder( + GetMetaURLQueryBuilder(page=page) + ) -@meta_url_router.put("/{meta_url_id}") +@meta_urls_router.put("/{url_id}") async def update_meta_url( - meta_url_id: int, + url_id: int, + request: UpdateMetaURLRequest, async_core: AsyncCore = Depends(get_async_core), - request: MetaURLUpdateRequest, ) -> MessageResponse: - return await async_core.adb_client.run_query_builder( - UpdateMetaURLQueryBuilder(meta_url_id=meta_url_id, meta_url_update=meta_url_update) + await check_is_meta_url(url_id=url_id, adb_client=async_core.adb_client) + await async_core.adb_client.run_query_builder( + UpdateMetaURLQueryBuilder( + url_id=url_id, + request=request + ) + ) + return MessageResponse(message="Meta URL updated.") + + +@meta_urls_router.get("/{url_id}/agencies") +async def get_meta_url_agencies( + url_id: int, + async_core: AsyncCore = Depends(get_async_core), +) -> AgencyGetOuterResponse: + return await get_meta_url_agencies_wrapper( + url_id=url_id, + adb_client=async_core.adb_client ) +@meta_urls_router.post("/{url_id}/agencies/{agency_id}") +async def add_agency_to_meta_url( + url_id: int, + agency_id: int, + async_core: AsyncCore = Depends(get_async_core), +) -> MessageResponse: + await add_meta_url_agency_link( + url_id=url_id, + agency_id=agency_id, + adb_client=async_core.adb_client + ) + return MessageResponse(message="Agency added to meta URL.") +@meta_urls_router.delete("/{url_id}/agencies/{agency_id}") +async def remove_agency_from_meta_url( + url_id: int, + agency_id: int, + async_core: AsyncCore = Depends(get_async_core), +) -> MessageResponse: + await delete_meta_url_agency_link( + url_id=url_id, + agency_id=agency_id, + adb_client=async_core.adb_client + ) + return MessageResponse(message="Agency removed from meta URL.") diff --git a/src/api/main.py b/src/api/main.py index 076b8108..2dd7fa24 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -12,7 +12,8 @@ from src.api.endpoints.batch.routes import batch_router from src.api.endpoints.collector.routes import collector_router from src.api.endpoints.contributions.routes import contributions_router -from src.api.endpoints.data_source.routes import data_source_router +from src.api.endpoints.data_source.routes import data_sources_router +from src.api.endpoints.meta_url.routes import meta_urls_router from src.api.endpoints.metrics.routes import metrics_router from src.api.endpoints.root import root_router from src.api.endpoints.search.routes import search_router @@ -181,7 +182,8 @@ async def redirect_docs(): submit_router, contributions_router, agencies_router, - data_source_router + data_sources_router, + meta_urls_router ] for router in routers: diff --git a/src/api/shared/agency/README.md b/src/api/shared/agency/README.md new file mode 100644 index 00000000..6afa1917 --- /dev/null +++ b/src/api/shared/agency/README.md @@ -0,0 +1 @@ +Logic for adding, removing and getting agencies by URL id \ No newline at end of file diff --git a/src/api/shared/agency/__init__.py b/src/api/shared/agency/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/agency/delete/__init__.py b/src/api/shared/agency/delete/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/agency/delete/query.py b/src/api/shared/agency/delete/query.py new file mode 100644 index 00000000..ca291a6f --- /dev/null +++ b/src/api/shared/agency/delete/query.py @@ -0,0 +1,29 @@ +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import AsyncSession + +from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency +from src.db.queries.base.builder import QueryBuilderBase + + +class RemoveURLAgencyLinkQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int, + agency_id: int + ): + super().__init__() + self.url_id = url_id + self.agency_id = agency_id + + async def run(self, session: AsyncSession) -> None: + statement = ( + delete( + LinkURLAgency + ) + .where( + LinkURLAgency.url_id == self.url_id, + LinkURLAgency.agency_id == self.agency_id + ) + ) + await session.execute(statement) \ No newline at end of file diff --git a/src/api/shared/agency/get/__init__.py b/src/api/shared/agency/get/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/agency/get/query.py b/src/api/shared/agency/get/query.py new file mode 100644 index 00000000..b49e47ee --- /dev/null +++ b/src/api/shared/agency/get/query.py @@ -0,0 +1,62 @@ +from typing import Sequence + +from sqlalchemy import select, RowMapping +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from src.api.endpoints.agencies.by_id.locations.get.response import AgencyGetLocationsResponse +from src.api.endpoints.agencies.root.get.response import AgencyGetResponse, AgencyGetOuterResponse +from src.db.models.impl.agency.sqlalchemy import Agency +from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency +from src.db.queries.base.builder import QueryBuilderBase + + +class GetRelatedAgenciesQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int + ): + super().__init__() + self.url_id = url_id + + async def run(self, session: AsyncSession) -> AgencyGetOuterResponse: + query = ( + select( + Agency, + ) + .options( + selectinload(Agency.locations) + ) + .join( + LinkURLAgency, + LinkURLAgency.agency_id == Agency.agency_id + ) + .where( + LinkURLAgency.url_id == self.url_id + ) + ) + + results: Sequence[RowMapping] = await self.sh.mappings( + session, + query=query + ) + responses: list[AgencyGetResponse] = [] + for result in results: + agency: Agency = result[Agency] + locations: list[AgencyGetLocationsResponse] = [ + AgencyGetLocationsResponse( + location_id=location.id, + full_display_name=location.full_display_name, + ) + for location in agency.locations + ] + responses.append(AgencyGetResponse( + id=agency.agency_id, + name=agency.name, + type=agency.agency_type, + jurisdiction_type=agency.jurisdiction_type, + locations=locations, + )) + + return AgencyGetOuterResponse(results=responses) diff --git a/src/api/shared/agency/get/response.py b/src/api/shared/agency/get/response.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/agency/post/__init__.py b/src/api/shared/agency/post/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/agency/post/query.py b/src/api/shared/agency/post/query.py new file mode 100644 index 00000000..045d1c84 --- /dev/null +++ b/src/api/shared/agency/post/query.py @@ -0,0 +1,32 @@ +from fastapi import HTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency +from src.db.queries.base.builder import QueryBuilderBase + + +class AddURLAgencyLinkQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int, + agency_id: int + ): + super().__init__() + self.url_id = url_id + self.agency_id = agency_id + + async def run(self, session: AsyncSession) -> None: + link = LinkURLAgency( + url_id=self.url_id, + agency_id=self.agency_id + ) + session.add(link) + try: + await session.commit() + except Exception as e: + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to add URL agency link: {e}" + ) \ No newline at end of file diff --git a/src/api/shared/batch/__init__.py b/src/api/shared/batch/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/batch/url/__init__.py b/src/api/shared/batch/url/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/batch/url/link.py b/src/api/shared/batch/url/link.py new file mode 100644 index 00000000..2ea22525 --- /dev/null +++ b/src/api/shared/batch/url/link.py @@ -0,0 +1,36 @@ +from fastapi import HTTPException +from sqlalchemy import delete +from sqlalchemy.ext.asyncio import AsyncSession + +from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL +from src.db.queries.base.builder import QueryBuilderBase + + +class UpdateBatchURLLinkQueryBuilder(QueryBuilderBase): + + def __init__( + self, + batch_id: int, + url_id: int + ): + super().__init__() + self.batch_id = batch_id + self.url_id = url_id + + async def run(self, session: AsyncSession) -> None: + + # Delete existing link if it exists + statement = ( + delete(LinkBatchURL) + .where( + LinkBatchURL.url_id==self.url_id + ) + ) + await session.execute(statement) + + # Add new link + link = LinkBatchURL( + batch_id=self.batch_id, + url_id=self.url_id + ) + session.add(link) diff --git a/src/api/shared/check/__init__.py b/src/api/shared/check/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/check/url_type/__init__.py b/src/api/shared/check/url_type/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/check/url_type/query.py b/src/api/shared/check/url_type/query.py new file mode 100644 index 00000000..be6287c2 --- /dev/null +++ b/src/api/shared/check/url_type/query.py @@ -0,0 +1,58 @@ +from fastapi import HTTPException +from sqlalchemy import select, RowMapping +from sqlalchemy.ext.asyncio import AsyncSession + +from src.db.models.impl.flag.url_validated.enums import URLType +from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.queries.base.builder import QueryBuilderBase + + +class CheckURLTypeQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int, + url_type: URLType + ): + super().__init__() + self.url_id = url_id + self.url_type = url_type + + async def run(self, session: AsyncSession) -> None: + """ + Raises: + Bad Request if url_type is not valid or does not exist + """ + + query = ( + select( + URL.id, + FlagURLValidated.type + ) + .outerjoin( + FlagURLValidated, + FlagURLValidated.url_id == URL.id + ) + .where( + URL.id == self.url_id, + ) + ) + + result: RowMapping | None = await self.sh.mapping(session, query=query) + if result is None: + raise HTTPException( + status_code=404, + detail="URL not found" + ) + url_type: URLType | None = result.get("type") + if url_type is None: + raise HTTPException( + status_code=400, + detail="URL is not validated" + ) + if url_type != self.url_type: + raise HTTPException( + status_code=400, + detail="URL type does not match expected URL type" + ) \ No newline at end of file diff --git a/src/api/shared/record_type/__init__.py b/src/api/shared/record_type/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/record_type/put/__init__.py b/src/api/shared/record_type/put/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/record_type/put/query.py b/src/api/shared/record_type/put/query.py new file mode 100644 index 00000000..f4cbae5c --- /dev/null +++ b/src/api/shared/record_type/put/query.py @@ -0,0 +1,32 @@ +from sqlalchemy import update +from sqlalchemy.ext.asyncio import AsyncSession + +from src.core.enums import RecordType +from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType +from src.db.queries.base.builder import QueryBuilderBase + + +class UpdateRecordTypeQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int, + record_type: RecordType + ): + super().__init__() + self.url_id = url_id + self.record_type = record_type + + async def run(self, session: AsyncSession) -> None: + statement = ( + update( + URLRecordType + ) + .where( + URLRecordType.url_id == self.url_id + ) + .values( + record_type=self.record_type + ) + ) + await session.execute(statement) \ No newline at end of file diff --git a/src/api/shared/url/__init__.py b/src/api/shared/url/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/url/put/__init__.py b/src/api/shared/url/put/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/shared/url/put/query.py b/src/api/shared/url/put/query.py new file mode 100644 index 00000000..a47a382c --- /dev/null +++ b/src/api/shared/url/put/query.py @@ -0,0 +1,50 @@ +from typing import Any + +from sqlalchemy import update +from sqlalchemy.ext.asyncio import AsyncSession + +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.queries.base.builder import QueryBuilderBase +from src.util.models.full_url import FullURL + + +class UpdateURLQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url_id: int, + url: str | None, + name: str | None, + description: str | None + ): + super().__init__() + self.url_id = url_id + self.url = url + self.name = name + self.description = description + + async def run(self, session: AsyncSession) -> Any: + values_dict = {} + if self.url is not None: + full_url = FullURL(self.url) + values_dict["url"] = full_url.id_form + values_dict["scheme"] = full_url.scheme + values_dict["trailing_slash"] = full_url.has_trailing_slash + if self.name is not None: + values_dict["name"] = self.name + if self.description is not None: + values_dict["description"] = self.description + + query = ( + update( + URL + ) + .where( + URL.id == self.url_id + ) + .values( + values_dict + ) + ) + + await session.execute(query) \ No newline at end of file diff --git a/src/db/models/impl/url/core/sqlalchemy.py b/src/db/models/impl/url/core/sqlalchemy.py index 50fa1676..02d4fbf2 100644 --- a/src/db/models/impl/url/core/sqlalchemy.py +++ b/src/db/models/impl/url/core/sqlalchemy.py @@ -21,7 +21,7 @@ class URL(UpdatedAtMixin, CreatedAtMixin, WithIDBase): # The batch this URL is associated with url = Column(Text, unique=True) - scheme = Column(String) + scheme: Mapped[str | None] = Column(String, nullable=True) name = Column(String) description = Column(Text) # The metadata from the collector diff --git a/tests/automated/integration/api/data_sources/__init__.py b/tests/automated/integration/api/data_sources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/api/data_sources/agencies/__init__.py b/tests/automated/integration/api/data_sources/agencies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/api/data_sources/agencies/test_add_remove.py b/tests/automated/integration/api/data_sources/agencies/test_add_remove.py new file mode 100644 index 00000000..7223c8ce --- /dev/null +++ b/tests/automated/integration/api/data_sources/agencies/test_add_remove.py @@ -0,0 +1,26 @@ +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.link.url_agency.sqlalchemy import LinkURLAgency +from tests.helpers.api_test_helper import APITestHelper + + +async def test_agencies_add_remove( + api_test_helper: APITestHelper, + test_url_data_source_id: int, + test_agency_id: int +): + api_test_helper.request_validator.post_v3( + url=f"/data-sources/{test_url_data_source_id}/agencies/{test_agency_id}", + ) + adb_client: AsyncDatabaseClient = api_test_helper.adb_client() + + links: list[LinkURLAgency] = await adb_client.get_all(LinkURLAgency) + assert len(links) == 1 + assert links[0].agency_id == test_agency_id + assert links[0].url_id == test_url_data_source_id + + api_test_helper.request_validator.delete_v3( + url=f"/data-sources/{test_url_data_source_id}/agencies/{test_agency_id}", + ) + + links: list[LinkURLAgency] = await adb_client.get_all(LinkURLAgency) + assert len(links) == 0 \ No newline at end of file diff --git a/tests/automated/integration/api/data_sources/agencies/test_invalid_type.py b/tests/automated/integration/api/data_sources/agencies/test_invalid_type.py new file mode 100644 index 00000000..54be1750 --- /dev/null +++ b/tests/automated/integration/api/data_sources/agencies/test_invalid_type.py @@ -0,0 +1,18 @@ +import pytest + +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.check import check_forbidden_url_type + + +@pytest.mark.asyncio +async def test_invalid_type( + api_test_helper: APITestHelper, + test_url_meta_url_id: int, + test_agency_id: int +): + for method in ['POST', 'DELETE']: + check_forbidden_url_type( + method=method, + route=f"/data-sources/{test_url_meta_url_id}/agencies/{test_agency_id}", + api_test_helper=api_test_helper, + ) \ No newline at end of file diff --git a/tests/automated/integration/api/data_sources/test_invalid_type.py b/tests/automated/integration/api/data_sources/test_invalid_type.py new file mode 100644 index 00000000..f415ee2b --- /dev/null +++ b/tests/automated/integration/api/data_sources/test_invalid_type.py @@ -0,0 +1,20 @@ +import pytest + +from src.api.endpoints.data_source.by_id.put.request import DataSourcePutRequest +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.check import check_forbidden_url_type + + +@pytest.mark.asyncio +async def test_invalid_type( + api_test_helper: APITestHelper, + test_url_meta_url_id: int +): + check_forbidden_url_type( + method="PUT", + route=f"/data-sources/{test_url_meta_url_id}", + api_test_helper=api_test_helper, + json=DataSourcePutRequest( + name="test" + ).model_dump(mode='json') + ) \ No newline at end of file diff --git a/tests/automated/integration/api/data_sources/test_put.py b/tests/automated/integration/api/data_sources/test_put.py new file mode 100644 index 00000000..c954b59c --- /dev/null +++ b/tests/automated/integration/api/data_sources/test_put.py @@ -0,0 +1,89 @@ +from datetime import date + +import pytest + +from src.api.endpoints.data_source.by_id.put.request import DataSourcePutRequest +from src.core.enums import RecordType +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \ + RetentionScheduleEnum, AccessTypeEnum +from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata +from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType +from tests.helpers.api_test_helper import APITestHelper + + +@pytest.mark.asyncio +async def test_put( + api_test_helper: APITestHelper, + test_url_data_source_id: int, + test_batch_id: int +): + + api_test_helper.request_validator.put_v3( + url=f"/data-sources/{test_url_data_source_id}", + json=DataSourcePutRequest( + url="http://modified_url.com/", + name="Modified URL", + record_type=RecordType.OTHER, + + batch_id=test_batch_id, + description="Modified Description", + + record_formats=["csv", "pdf"], + data_portal_type="CKAN", + supplying_entity="Modified Supplying Entity", + coverage_start=date(year=2025, month=4, day=1), + coverage_end=date(year=2025, month=8, day=29), + agency_supplied=False, + agency_originated=True, + agency_aggregation=AgencyAggregationEnum.LOCALITY, + agency_described_not_in_database="Modified Agency Not In DB", + update_method=UpdateMethodEnum.OVERWRITE, + readme_url="https://modified-readme.com", + originating_entity="Modified Originating Entity", + retention_schedule=RetentionScheduleEnum.FUTURE_ONLY, + scraper_url="https://modified-scraper.com", + submission_notes="Modified Submission Notes", + access_notes="Modified Access Notes", + access_types=[AccessTypeEnum.WEBPAGE, AccessTypeEnum.API], + ).model_dump(mode='json') + + ) + + adb_client: AsyncDatabaseClient = api_test_helper.adb_client() + + url: URL = (await adb_client.get_all(URL))[0] + assert url.url == "modified_url.com" + assert url.scheme == "http" + assert url.trailing_slash == True + assert url.description == "Modified Description" + + # Check Record Type + record_type: URLRecordType = (await adb_client.get_all(URLRecordType))[0] + assert record_type.record_type == RecordType.OTHER + + # Check Batch Link + link: LinkBatchURL = (await adb_client.get_all(LinkBatchURL))[0] + assert link.batch_id == test_batch_id + + # Check Optional Metadata + optional_metadata: URLOptionalDataSourceMetadata = (await adb_client.get_all(URLOptionalDataSourceMetadata))[0] + assert optional_metadata.record_formats == ["csv", "pdf"] + assert optional_metadata.data_portal_type == "CKAN" + assert optional_metadata.supplying_entity == "Modified Supplying Entity" + assert optional_metadata.coverage_start == date(year=2025, month=4, day=1) + assert optional_metadata.coverage_end == date(year=2025, month=8, day=29) + assert optional_metadata.agency_supplied == False + assert optional_metadata.agency_originated == True + assert optional_metadata.agency_aggregation == AgencyAggregationEnum.LOCALITY + assert optional_metadata.agency_described_not_in_database == "Modified Agency Not In DB" + assert optional_metadata.update_method == UpdateMethodEnum.OVERWRITE + assert optional_metadata.readme_url == "https://modified-readme.com" + assert optional_metadata.originating_entity == "Modified Originating Entity" + assert optional_metadata.retention_schedule == RetentionScheduleEnum.FUTURE_ONLY + assert optional_metadata.scraper_url == "https://modified-scraper.com" + assert optional_metadata.submission_notes == "Modified Submission Notes" + assert optional_metadata.access_notes == "Modified Access Notes" + assert optional_metadata.access_types == [AccessTypeEnum.WEBPAGE, AccessTypeEnum.API] diff --git a/tests/automated/integration/api/meta_urls/__init__.py b/tests/automated/integration/api/meta_urls/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/api/meta_urls/agencies/__init__.py b/tests/automated/integration/api/meta_urls/agencies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/api/meta_urls/agencies/test_add_remove.py b/tests/automated/integration/api/meta_urls/agencies/test_add_remove.py new file mode 100644 index 00000000..4f48ac5c --- /dev/null +++ b/tests/automated/integration/api/meta_urls/agencies/test_add_remove.py @@ -0,0 +1,30 @@ +from src.api.endpoints.agencies.root.get.response import AgencyGetOuterResponse +from tests.helpers.api_test_helper import APITestHelper + + +async def test_agencies_add_remove( + api_test_helper: APITestHelper, + test_url_meta_url_id: int, + test_agency_id: int +): + api_test_helper.request_validator.post_v3( + url=f"/meta-urls/{test_url_meta_url_id}/agencies/{test_agency_id}", + ) + + raw_response: dict = api_test_helper.request_validator.get_v3( + url=f"/meta-urls/{test_url_meta_url_id}/agencies", + ) + response = AgencyGetOuterResponse(**raw_response) + assert len(response.results) == 1 + assert response.results[0].id == test_agency_id + + + api_test_helper.request_validator.delete_v3( + url=f"/meta-urls/{test_url_meta_url_id}/agencies/{test_agency_id}", + ) + + raw_response: dict = api_test_helper.request_validator.get_v3( + url=f"/meta-urls/{test_url_meta_url_id}/agencies", + ) + response = AgencyGetOuterResponse(**raw_response) + assert len(response.results) == 0 diff --git a/tests/automated/integration/api/meta_urls/agencies/test_invalid_type.py b/tests/automated/integration/api/meta_urls/agencies/test_invalid_type.py new file mode 100644 index 00000000..4f3c6f4a --- /dev/null +++ b/tests/automated/integration/api/meta_urls/agencies/test_invalid_type.py @@ -0,0 +1,18 @@ +import pytest + +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.check import check_forbidden_url_type + + +@pytest.mark.asyncio +async def test_invalid_type( + api_test_helper: APITestHelper, + test_url_data_source_id: int, + test_agency_id: int +): + for method in ['POST', 'DELETE']: + check_forbidden_url_type( + method=method, + route=f"/meta-urls/{test_url_data_source_id}/agencies/{test_agency_id}", + api_test_helper=api_test_helper, + ) \ No newline at end of file diff --git a/tests/automated/integration/api/meta_urls/test_invalid_type.py b/tests/automated/integration/api/meta_urls/test_invalid_type.py new file mode 100644 index 00000000..12073191 --- /dev/null +++ b/tests/automated/integration/api/meta_urls/test_invalid_type.py @@ -0,0 +1,20 @@ +import pytest + +from src.api.endpoints.meta_url.by_id.agencies.put.request import UpdateMetaURLRequest +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.check import check_forbidden_url_type + + +@pytest.mark.asyncio +async def test_invalid_type( + api_test_helper: APITestHelper, + test_url_data_source_id: int +): + check_forbidden_url_type( + method="PUT", + route=f"/meta-urls/{test_url_data_source_id}", + api_test_helper=api_test_helper, + json=UpdateMetaURLRequest( + name="test" + ).model_dump(mode='json') + ) \ No newline at end of file diff --git a/tests/automated/integration/api/meta_urls/test_put.py b/tests/automated/integration/api/meta_urls/test_put.py new file mode 100644 index 00000000..28689a8b --- /dev/null +++ b/tests/automated/integration/api/meta_urls/test_put.py @@ -0,0 +1,39 @@ +import pytest + +from src.api.endpoints.meta_url.by_id.agencies.put.request import UpdateMetaURLRequest +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.link.batch_url.sqlalchemy import LinkBatchURL +from src.db.models.impl.url.core.sqlalchemy import URL +from tests.helpers.api_test_helper import APITestHelper + + +@pytest.mark.asyncio +async def test_put( + api_test_helper: APITestHelper, + test_url_meta_url_id: int, + test_batch_id: int +): + api_test_helper.request_validator.put_v3( + url=f"/meta-urls/{test_url_meta_url_id}", + json=UpdateMetaURLRequest( + url="new-meta-url.com", + name="Modified name", + description="Modified description", + batch_id=test_batch_id + ).model_dump(mode='json') + ) + + adb_client: AsyncDatabaseClient = api_test_helper.adb_client() + + # Check URL updated (including schema and trailing slash) + url: URL = (await adb_client.get_all(URL))[0] + assert url.url == "new-meta-url.com" + assert url.name == "Modified name" + assert url.scheme == "" + assert url.trailing_slash == False + assert url.description == "Modified description" + + # Check Batch ID + link: LinkBatchURL = (await adb_client.get_all(LinkBatchURL))[0] + assert link.batch_id == test_batch_id + diff --git a/tests/automated/integration/conftest.py b/tests/automated/integration/conftest.py index b4466424..42ab2214 100644 --- a/tests/automated/integration/conftest.py +++ b/tests/automated/integration/conftest.py @@ -8,9 +8,11 @@ from src.api.main import app from src.collectors.manager import AsyncCollectorManager from src.core.core import AsyncCore +from src.core.enums import RecordType from src.core.logger import AsyncCoreLogger from src.db.client.async_ import AsyncDatabaseClient from src.db.client.sync import DatabaseClient +from src.db.models.impl.flag.url_validated.enums import URLType from src.security.dtos.access_info import AccessInfo from src.security.enums import Permissions from src.security.manager import get_access_info @@ -160,10 +162,33 @@ async def api_test_helper( ) await client.app.state.async_core.collector_manager.logger.clear_log_queue() +@pytest.fixture +def test_batch_id( + db_data_creator: DBDataCreator +) -> int: + return db_data_creator.batch() + @pytest_asyncio.fixture async def test_agency_id( db_data_creator: DBDataCreator ) -> int: return await db_data_creator.agency( name="Test Agency" - ) \ No newline at end of file + ) + +@pytest_asyncio.fixture +async def test_url_data_source_id( + db_data_creator: DBDataCreator +) -> int: + return (await db_data_creator.create_validated_urls( + record_type=RecordType.CRIME_STATISTICS, + validation_type=URLType.DATA_SOURCE, + ))[0].url_id + +@pytest_asyncio.fixture +async def test_url_meta_url_id( + db_data_creator: DBDataCreator +) -> int: + return (await db_data_creator.create_validated_urls( + validation_type=URLType.META_URL, + ))[0].url_id diff --git a/tests/automated/integration/readonly/api/agencies/get/test_locations.py b/tests/automated/integration/readonly/api/agencies/get/test_locations.py index 13481c58..34904057 100644 --- a/tests/automated/integration/readonly/api/agencies/get/test_locations.py +++ b/tests/automated/integration/readonly/api/agencies/get/test_locations.py @@ -1,6 +1,6 @@ import pytest -from tests.automated.integration.readonly.conftest import ReadOnlyTestHelper +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper @pytest.mark.asyncio diff --git a/tests/automated/integration/readonly/api/agencies/get/test_root.py b/tests/automated/integration/readonly/api/agencies/get/test_root.py index fa390abd..a74e49da 100644 --- a/tests/automated/integration/readonly/api/agencies/get/test_root.py +++ b/tests/automated/integration/readonly/api/agencies/get/test_root.py @@ -1,7 +1,7 @@ import pytest from src.db.models.impl.agency.enums import JurisdictionType, AgencyType -from tests.automated.integration.readonly.conftest import ReadOnlyTestHelper +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper @pytest.mark.asyncio diff --git a/tests/automated/integration/readonly/api/data_sources/__init__.py b/tests/automated/integration/readonly/api/data_sources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/readonly/api/data_sources/agencies/__init__.py b/tests/automated/integration/readonly/api/data_sources/agencies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/readonly/api/data_sources/agencies/test_forbid.py b/tests/automated/integration/readonly/api/data_sources/agencies/test_forbid.py new file mode 100644 index 00000000..85a54705 --- /dev/null +++ b/tests/automated/integration/readonly/api/data_sources/agencies/test_forbid.py @@ -0,0 +1,13 @@ +import pytest + +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper +from tests.helpers.check import check_forbidden_url_type + + +@pytest.mark.asyncio +async def test_forbid(readonly_helper: ReadOnlyTestHelper): + check_forbidden_url_type( + route=f"/data-sources/{readonly_helper.url_meta_url_id}/agencies", + api_test_helper=readonly_helper.api_test_helper, + method="GET" + ) diff --git a/tests/automated/integration/readonly/api/data_sources/test_get.py b/tests/automated/integration/readonly/api/data_sources/test_get.py new file mode 100644 index 00000000..e7bbe861 --- /dev/null +++ b/tests/automated/integration/readonly/api/data_sources/test_get.py @@ -0,0 +1,57 @@ +from datetime import date + +import pytest +from deepdiff import DeepDiff + +from src.api.endpoints.data_source.get.response import DataSourceGetOuterResponse, DataSourceGetResponse +from src.core.enums import RecordType +from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \ + RetentionScheduleEnum, AccessTypeEnum +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper + + +@pytest.mark.asyncio +async def test_get(readonly_helper: ReadOnlyTestHelper): + + raw_json: dict = readonly_helper.api_test_helper.request_validator.get_v3( + url=f"/data-sources", + ) + outer_response = DataSourceGetOuterResponse(**raw_json) + + assert len(outer_response.results) == 1 + response: DataSourceGetResponse = outer_response.results[0] + + diff = DeepDiff( + response.model_dump(mode='json'), + DataSourceGetResponse( + url_id=readonly_helper.url_data_source_id, + url="read-only-ds.com", + + name="Read only URL name", + record_type=RecordType.CRIME_STATISTICS, + agency_ids=[readonly_helper.agency_1_id], + + batch_id=None, + description="Read only URL", + + record_formats=["csv", "pdf"], + data_portal_type="CKAN", + supplying_entity="ReadOnly Agency", + coverage_start=date(year=2025, month=6, day=1), + coverage_end=date(year=2025, month=8, day=20), + agency_supplied=False, + agency_originated=True, + agency_aggregation=AgencyAggregationEnum.LOCALITY, + agency_described_not_in_database="ReadOnly Agency Not In DB", + update_method=UpdateMethodEnum.NO_UPDATES, + readme_url="https://read-only-readme.com", + originating_entity="ReadOnly Agency Originating", + retention_schedule=RetentionScheduleEnum.GT_10_YEARS, + scraper_url="https://read-only-scraper.com", + submission_notes="Read Only Submission Notes", + access_notes="Read Only Access Notes", + access_types=[AccessTypeEnum.WEBPAGE, AccessTypeEnum.API], + ).model_dump(mode='json'), + ) + + assert diff == {}, f"Differences found: {diff}" diff --git a/tests/automated/integration/readonly/api/meta_urls/__init__.py b/tests/automated/integration/readonly/api/meta_urls/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/readonly/api/meta_urls/agencies/__init__.py b/tests/automated/integration/readonly/api/meta_urls/agencies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/automated/integration/readonly/api/meta_urls/agencies/test_forbid.py b/tests/automated/integration/readonly/api/meta_urls/agencies/test_forbid.py new file mode 100644 index 00000000..d62fa524 --- /dev/null +++ b/tests/automated/integration/readonly/api/meta_urls/agencies/test_forbid.py @@ -0,0 +1,15 @@ + +import pytest + +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper +from tests.helpers.check import check_forbidden_url_type + + +@pytest.mark.asyncio +async def test_forbid(readonly_helper: ReadOnlyTestHelper): + check_forbidden_url_type( + route=f"/meta-urls/{readonly_helper.url_data_source_id}/agencies", + api_test_helper=readonly_helper.api_test_helper, + method="GET" + ) + diff --git a/tests/automated/integration/readonly/api/meta_urls/test_get.py b/tests/automated/integration/readonly/api/meta_urls/test_get.py new file mode 100644 index 00000000..8779a3fc --- /dev/null +++ b/tests/automated/integration/readonly/api/meta_urls/test_get.py @@ -0,0 +1,30 @@ +import pytest +from deepdiff import DeepDiff + +from src.api.endpoints.meta_url.get.response import MetaURLGetOuterResponse, MetaURLGetResponse +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper + + +@pytest.mark.asyncio +async def test_get(readonly_helper: ReadOnlyTestHelper): + + raw_json: dict = readonly_helper.api_test_helper.request_validator.get_v3( + url=f"/meta-urls", + ) + outer_response = MetaURLGetOuterResponse(**raw_json) + + assert len(outer_response.results) == 1 + response: MetaURLGetResponse = outer_response.results[0] + + diff = DeepDiff( + response.model_dump(mode='json'), + MetaURLGetResponse( + url_id=readonly_helper.url_meta_url_id, + url="read-only-meta-url.com", + name="Read only URL Name", + description="Read only URL", + batch_id=None, + agency_ids=[] + ).model_dump(mode='json'), + ) + assert diff == {}, f"Differences found: {diff}" \ No newline at end of file diff --git a/tests/automated/integration/readonly/conftest.py b/tests/automated/integration/readonly/conftest.py index 1085c184..a5bcd249 100644 --- a/tests/automated/integration/readonly/conftest.py +++ b/tests/automated/integration/readonly/conftest.py @@ -3,33 +3,19 @@ import pytest import pytest_asyncio -from pydantic import BaseModel from starlette.testclient import TestClient from src.db.client.async_ import AsyncDatabaseClient from src.db.helpers.connect import get_postgres_connection_string -from src.db.models.impl.agency.enums import AgencyType, JurisdictionType -from src.db.models.impl.agency.sqlalchemy import Agency -from src.db.models.impl.link.agency_location.sqlalchemy import LinkAgencyLocation from tests.automated.integration.api._helpers.RequestValidator import RequestValidator +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper +from tests.automated.integration.readonly.setup import setup_readonly_data from tests.helpers.api_test_helper import APITestHelper -from tests.helpers.counter import next_int from tests.helpers.data_creator.core import DBDataCreator from tests.helpers.data_creator.models.creation_info.us_state import USStateCreationInfo from tests.helpers.setup.wipe import wipe_database -class ReadOnlyTestHelper(BaseModel): - class Config: - arbitrary_types_allowed = True - - adb_client: AsyncDatabaseClient - api_test_helper: APITestHelper - - agency_1_id: int - agency_1_location_id: int - - @pytest.fixture(scope="module") def event_loop(): loop = asyncio.new_event_loop() @@ -50,8 +36,6 @@ async def readonly_helper( client: TestClient, ) -> AsyncGenerator[ReadOnlyTestHelper, Any]: wipe_database(get_postgres_connection_string()) - conn = get_postgres_connection_string(is_async=True) - adb_client = AsyncDatabaseClient(db_url=conn) db_data_creator = DBDataCreator() api_test_helper = APITestHelper( request_validator=RequestValidator(client=client), @@ -59,43 +43,6 @@ async def readonly_helper( db_data_creator=db_data_creator, ) - # Pennsylvania - pennsylvania = await DBDataCreator().create_us_state( - name="Pennsylvania", - iso="PA" - ) - - allegheny_county = await DBDataCreator().create_county( - state_id=pennsylvania.us_state_id, - name="Allegheny" - ) - pittsburgh = await DBDataCreator().create_locality( - state_id=pennsylvania.us_state_id, - county_id=allegheny_county.county_id, - name="Pittsburgh" - ) - - - # Add Agencies - agency_1 = Agency( - agency_id=next_int(), - name="Agency 1", - agency_type=AgencyType.LAW_ENFORCEMENT, - jurisdiction_type=JurisdictionType.STATE, - ) - await adb_client.add(agency_1) - - # Add Agency location - agency_1_location = LinkAgencyLocation( - agency_id=agency_1.agency_id, - location_id=pittsburgh.location_id, - ) - await adb_client.add(agency_1_location) - - yield ReadOnlyTestHelper( - adb_client=adb_client, - api_test_helper=api_test_helper, + helper: ReadOnlyTestHelper = await setup_readonly_data(api_test_helper=api_test_helper) - agency_1_id=agency_1.agency_id, - agency_1_location_id=pittsburgh.location_id, - ) \ No newline at end of file + yield helper \ No newline at end of file diff --git a/tests/automated/integration/readonly/helper.py b/tests/automated/integration/readonly/helper.py new file mode 100644 index 00000000..68474256 --- /dev/null +++ b/tests/automated/integration/readonly/helper.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel + +from src.db.client.async_ import AsyncDatabaseClient +from tests.helpers.api_test_helper import APITestHelper + + +class ReadOnlyTestHelper(BaseModel): + class Config: + arbitrary_types_allowed = True + + adb_client: AsyncDatabaseClient + api_test_helper: APITestHelper + + agency_1_id: int + agency_1_location_id: int + + url_data_source_id: int + url_meta_url_id: int diff --git a/tests/automated/integration/readonly/setup.py b/tests/automated/integration/readonly/setup.py new file mode 100644 index 00000000..20c6d537 --- /dev/null +++ b/tests/automated/integration/readonly/setup.py @@ -0,0 +1,171 @@ +from datetime import date + +from src.collectors.enums import URLStatus +from src.core.enums import RecordType +from src.db.client.async_ import AsyncDatabaseClient +from src.db.models.impl.agency.enums import AgencyType, JurisdictionType +from src.db.models.impl.agency.sqlalchemy import Agency +from src.db.models.impl.flag.url_validated.enums import URLType +from src.db.models.impl.link.agency_location.sqlalchemy import LinkAgencyLocation +from src.db.models.impl.url.core.enums import URLSource +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.models.impl.url.optional_ds_metadata.enums import AgencyAggregationEnum, UpdateMethodEnum, \ + RetentionScheduleEnum, AccessTypeEnum +from src.db.models.impl.url.optional_ds_metadata.sqlalchemy import URLOptionalDataSourceMetadata +from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType +from tests.automated.integration.readonly.helper import ReadOnlyTestHelper +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.counter import next_int +from tests.helpers.data_creator.core import DBDataCreator +from tests.helpers.data_creator.models.creation_info.county import CountyCreationInfo +from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo +from tests.helpers.data_creator.models.creation_info.us_state import USStateCreationInfo + + +async def setup_readonly_data( + api_test_helper: APITestHelper +) -> ReadOnlyTestHelper: + db_data_creator = api_test_helper.db_data_creator + adb_client = db_data_creator.adb_client + + # Pennsylvania + pennsylvania: USStateCreationInfo = await db_data_creator.create_us_state( + name="Pennsylvania", + iso="PA" + ) + + allegheny_county: CountyCreationInfo = await db_data_creator.create_county( + state_id=pennsylvania.us_state_id, + name="Allegheny" + ) + pittsburgh: LocalityCreationInfo = await db_data_creator.create_locality( + state_id=pennsylvania.us_state_id, + county_id=allegheny_county.county_id, + name="Pittsburgh" + ) + + + # Add Agencies + agency_1_id: int = await add_agency(adb_client, pittsburgh) + + # Add Data Source With Linked Agency + url_data_source_id: int = await add_data_source(agency_1_id, db_data_creator) + + # Add Meta URL with Linked Agency + url_meta_url_id: int = await add_meta_url(agency_1_id, db_data_creator) + + return ReadOnlyTestHelper( + adb_client=adb_client, + api_test_helper=api_test_helper, + + agency_1_id=agency_1_id, + agency_1_location_id=pittsburgh.location_id, + + url_data_source_id=url_data_source_id, + url_meta_url_id=url_meta_url_id, + ) + + +async def add_meta_url( + agency_1_id: int, + db_data_creator: DBDataCreator +) -> int: + adb_client: AsyncDatabaseClient = db_data_creator.adb_client + url = URL( + scheme=None, + url="read-only-meta-url.com", + name="Read only URL Name", + trailing_slash=False, + description="Read only URL", + collector_metadata={ + "url": "https://read-only-meta-url.com/" + }, + status=URLStatus.OK, + source=URLSource.REDIRECT, + ) + url_id: int = await adb_client.add(url, return_id=True) + + await db_data_creator.create_validated_flags( + url_ids=[url_id], + validation_type=URLType.META_URL + ) + + return url_id + + +async def add_data_source( + agency_1_id: int, + db_data_creator: DBDataCreator +) -> int: + adb_client: AsyncDatabaseClient = db_data_creator.adb_client + url = URL( + scheme="https", + url="read-only-ds.com", + name="Read only URL name", + trailing_slash=True, + description="Read only URL", + collector_metadata={ + "url": "https://read-only.com/" + }, + status=URLStatus.OK, + source=URLSource.COLLECTOR, + ) + url_id: int = await adb_client.add(url, return_id=True) + await db_data_creator.create_validated_flags( + url_ids=[url_id], + validation_type=URLType.DATA_SOURCE + ) + record_type = URLRecordType( + url_id=url_id, + record_type=RecordType.CRIME_STATISTICS + ) + await adb_client.add(record_type) + + optional_ds_metadata = URLOptionalDataSourceMetadata( + url_id=url_id, + record_formats=["csv", "pdf"], + data_portal_type="CKAN", + supplying_entity="ReadOnly Agency", + coverage_start=date(year=2025, month=6, day=1), + coverage_end=date(year=2025, month=8, day=20), + agency_supplied=False, + agency_originated=True, + agency_aggregation=AgencyAggregationEnum.LOCALITY, + agency_described_not_in_database="ReadOnly Agency Not In DB", + update_method=UpdateMethodEnum.NO_UPDATES, + readme_url="https://read-only-readme.com", + originating_entity="ReadOnly Agency Originating", + retention_schedule=RetentionScheduleEnum.GT_10_YEARS, + scraper_url="https://read-only-scraper.com", + submission_notes="Read Only Submission Notes", + access_notes="Read Only Access Notes", + access_types=[AccessTypeEnum.WEBPAGE, AccessTypeEnum.API], + ) + + await adb_client.add(optional_ds_metadata) + + await db_data_creator.create_url_agency_links( + url_ids=[url_id], + agency_ids=[agency_1_id] + ) + return url_id + + +async def add_agency( + adb_client: AsyncDatabaseClient, + pittsburgh: LocalityCreationInfo +) -> int: + agency_1 = Agency( + agency_id=next_int(), + name="Agency 1", + agency_type=AgencyType.LAW_ENFORCEMENT, + jurisdiction_type=JurisdictionType.STATE, + ) + await adb_client.add(agency_1) + # Add Agency location + agency_1_location = LinkAgencyLocation( + agency_id=agency_1.agency_id, + location_id=pittsburgh.location_id, + ) + await adb_client.add(agency_1_location) + return agency_1.agency_id \ No newline at end of file diff --git a/tests/automated/integration/tasks/url/impl/submit_approved/test_submit_approved_url_task.py b/tests/automated/integration/tasks/url/impl/submit_approved/test_submit_approved_url_task.py index 3d1aec23..22ae8129 100644 --- a/tests/automated/integration/tasks/url/impl/submit_approved/test_submit_approved_url_task.py +++ b/tests/automated/integration/tasks/url/impl/submit_approved/test_submit_approved_url_task.py @@ -2,7 +2,6 @@ from deepdiff import DeepDiff from pdap_access_manager import RequestInfo, RequestType, DataSourcesNamespaces -from src.collectors.enums import URLStatus from src.core.tasks.url.enums import TaskOperatorOutcome from src.core.tasks.url.operators.submit_approved.core import SubmitApprovedURLTaskOperator from src.db.models.impl.url.core.sqlalchemy import URL diff --git a/tests/helpers/awaitable_barrier.py b/tests/helpers/awaitable_barrier.py deleted file mode 100644 index 8bf65a11..00000000 --- a/tests/helpers/awaitable_barrier.py +++ /dev/null @@ -1,13 +0,0 @@ -import asyncio - - -class AwaitableBarrier: - def __init__(self): - self._event = asyncio.Event() - - async def __call__(self, *args, **kwargs): - await self._event.wait() - - def release(self): - self._event.set() - diff --git a/tests/helpers/check.py b/tests/helpers/check.py new file mode 100644 index 00000000..b9172151 --- /dev/null +++ b/tests/helpers/check.py @@ -0,0 +1,20 @@ +import pytest +from fastapi import HTTPException + +from tests.helpers.api_test_helper import APITestHelper + + +def check_forbidden_url_type( + route: str, + method: str, + api_test_helper: APITestHelper, + **kwargs +) -> None: + with pytest.raises(HTTPException) as e: + api_test_helper.request_validator.open_v3( + url=route, + method=method, + **kwargs + ) + assert e.value.status_code == 400, f"Expected status code 400, got {e.value.status_code}" + assert e.value.detail['detail'] == 'URL type does not match expected URL type' \ No newline at end of file diff --git a/tests/helpers/patch_functions.py b/tests/helpers/patch_functions.py deleted file mode 100644 index 170a2062..00000000 --- a/tests/helpers/patch_functions.py +++ /dev/null @@ -1,10 +0,0 @@ -from tests.helpers.awaitable_barrier import AwaitableBarrier - - -async def block_sleep(monkeypatch) -> AwaitableBarrier: - barrier = AwaitableBarrier() - monkeypatch.setattr( - "src.collectors.impl.example.core.ExampleCollector.sleep", - barrier - ) - return barrier