diff --git a/truecover-app/src/components/ExportLocationsModal.tsx b/truecover-app/src/components/ExportLocationsModal.tsx index 7fd769bfb..a5844dae4 100644 --- a/truecover-app/src/components/ExportLocationsModal.tsx +++ b/truecover-app/src/components/ExportLocationsModal.tsx @@ -42,9 +42,9 @@ const ExportLocationsModal: React.FC = ({ // Entity export workflow state const [exportWorkflowId, setExportWorkflowId] = useState(null); const [exportProgress, setExportProgress] = useState<{ - total_pixels: number; - created_pixels: number; - current_quadkey: string; + total_entities: number; + created_entities: number; + current_label: string; } | null>(null); const [isExporting, setIsExporting] = useState(false); const [pixelGeometryType, setPixelGeometryType] = useState<'centroid' | 'boundary'>('centroid'); @@ -748,8 +748,11 @@ const ExportLocationsModal: React.FC = ({ )} - {/* Pixel Geometry Type Selection */} -
+ {/* Pixel Geometry Type Selection - only shown when pixel rounds are selected */} + {selectedRoundIds.some(id => { + const round = rounds.find(r => r.id === id); + return round?.sampling_target === 'pixels'; + }) &&
@@ -787,7 +790,7 @@ const ExportLocationsModal: React.FC = ({
- + } )} @@ -826,12 +829,12 @@ const ExportLocationsModal: React.FC = ({
Progress: - {exportProgress.created_pixels} / {exportProgress.total_pixels} pixels + {exportProgress.created_entities} / {exportProgress.total_entities} entities
- {exportProgress.current_quadkey && ( + {exportProgress.current_label && (
- Current: {exportProgress.current_quadkey} + Current: {exportProgress.current_label}
)} diff --git a/truecover-app/src/components/ProjectSettings.tsx b/truecover-app/src/components/ProjectSettings.tsx index c93a23d9f..cac8205c2 100644 --- a/truecover-app/src/components/ProjectSettings.tsx +++ b/truecover-app/src/components/ProjectSettings.tsx @@ -63,6 +63,11 @@ const ProjectSettings: React.FC = ({ setError(null); setOnaProjects([]); setOnaEntityLists([]); + + // Auto-load entity lists when project is configured but no entity list is set + if (project.ona_project_id && !project.ona_entity_list_id) { + loadEntityListsForProject(project.ona_project_id); + } } }, [project, isOpen]); diff --git a/truecover-app/src/services/api.ts b/truecover-app/src/services/api.ts index 6fb755d5b..76153cb26 100644 --- a/truecover-app/src/services/api.ts +++ b/truecover-app/src/services/api.ts @@ -962,15 +962,15 @@ export const entityExportApi = { workflow_id: string; status: string; progress?: { - total_pixels: number; - created_pixels: number; - current_quadkey: string; + total_entities: number; + created_entities: number; + current_label: string; error_message: string | null; }; result?: { success: boolean; - total_pixels: number; - created_pixels: number; + total_entities: number; + created_entities: number; message: string; }; error?: string; diff --git a/truecover-backend/db/migrations/add_sampled_only_filter.sql b/truecover-backend/db/migrations/add_sampled_only_filter.sql index 29b98d8de..e340414d3 100644 --- a/truecover-backend/db/migrations/add_sampled_only_filter.sql +++ b/truecover-backend/db/migrations/add_sampled_only_filter.sql @@ -71,7 +71,7 @@ BEGIN SELECT p.quadkey, p.geometry, - (pm.metadata->>'population')::numeric AS population, + p.population, lc.building_count FROM pixels p JOIN pixel_area pa ON p.quadkey = pa.quadkey @@ -79,7 +79,6 @@ BEGIN LEFT JOIN coverage_pixel cp ON p.quadkey = cp.quadkey AND cp.campaign_id = target_campaign_id AND (target_indicator_id IS NULL OR cp.indicator_id = target_indicator_id) - LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey LEFT JOIN LATERAL ( SELECT COUNT(*)::integer AS building_count FROM locations l diff --git a/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql b/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql index 3e4d8f97d..b5523449a 100644 --- a/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql +++ b/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql @@ -64,12 +64,11 @@ BEGIN SELECT p.quadkey, p.geometry, - (pm.metadata->>'population')::numeric AS population, + p.population, lc.building_count FROM pixels p JOIN pixel_area pa ON p.quadkey = pa.quadkey JOIN campaign_areas ca ON pa.campaign_area_id = ca.id - LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey LEFT JOIN LATERAL ( SELECT COUNT(*)::integer AS building_count FROM locations l diff --git a/truecover-backend/routes/campaigns.py b/truecover-backend/routes/campaigns.py index c118f33fc..714ab0317 100644 --- a/truecover-backend/routes/campaigns.py +++ b/truecover-backend/routes/campaigns.py @@ -581,10 +581,9 @@ def compute_pixels_for_area(user, area_id): WITH pixel_stats AS ( SELECT COUNT(*) as pixel_count, - COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population + COALESCE(SUM(p.population), 0) as total_population FROM pixel_area pa JOIN pixels p ON pa.quadkey = p.quadkey - LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey WHERE pa.campaign_area_id = %s ), location_counts AS ( @@ -696,10 +695,9 @@ def compute_all_pixels_for_campaign(user, campaign_id): WITH pixel_stats AS ( SELECT COUNT(*) as pixel_count, - COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population + COALESCE(SUM(p.population), 0) as total_population FROM pixel_area pa JOIN pixels p ON pa.quadkey = p.quadkey - LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey WHERE pa.campaign_area_id = %s ), location_counts AS ( diff --git a/truecover-backend/routes/entity_export.py b/truecover-backend/routes/entity_export.py index f34441f29..69777b102 100644 --- a/truecover-backend/routes/entity_export.py +++ b/truecover-backend/routes/entity_export.py @@ -1,9 +1,10 @@ -# ABOUTME: Routes for exporting pixel coverage data to ODK entity lists +# ABOUTME: Routes for exporting coverage data (pixels and locations) to ODK entity lists # ABOUTME: Handles starting and monitoring Temporal workflows for entity creation from flask import Blueprint, jsonify, request from auth.middleware import require_auth from auth.helpers import check_campaign_access +from db.connection import get_db_connection, return_db_connection entity_export_bp = Blueprint('entity_export', __name__) @@ -39,6 +40,29 @@ def start_entity_export_workflow(user, campaign_id): if not project_id: return jsonify({'error': 'project_id is required'}), 400 + # Split rounds by sampling_target + conn = get_db_connection() + cursor = conn.cursor() + try: + placeholders = ','.join(['%s'] * len(round_ids)) + cursor.execute(f""" + SELECT id, COALESCE(sampling_target, 'locations') as sampling_target + FROM rounds + WHERE id IN ({placeholders}) + """, tuple(round_ids)) + + pixel_round_ids = [] + location_round_ids = [] + for row in cursor.fetchall(): + rid, target = str(row[0]), row[1] + if target == 'pixels': + pixel_round_ids.append(rid) + else: + location_round_ids.append(rid) + finally: + cursor.close() + return_db_connection(conn) + # Generate workflow ID timestamp = datetime.now().strftime('%Y%m%d%H%M%S') workflow_id = f"entity-export-{campaign_id}-{timestamp}" @@ -48,7 +72,7 @@ async def start_workflow(): client = await get_temporal_client() handle = await client.start_workflow( EntityExportWorkflow.run, - args=[campaign_id, indicator_id, round_ids, project_id, geometry_type], + args=[campaign_id, indicator_id, pixel_round_ids, location_round_ids, project_id, geometry_type], id=workflow_id, task_queue="truecover-tasks" ) diff --git a/truecover-backend/temporal/activities/enrichment.py b/truecover-backend/temporal/activities/enrichment.py index f14bd3cac..2f48d2236 100644 --- a/truecover-backend/temporal/activities/enrichment.py +++ b/truecover-backend/temporal/activities/enrichment.py @@ -341,6 +341,17 @@ async def enrich_area_pixels( metadata = pixel_metadata.metadata || EXCLUDED.metadata, updated_at = NOW() """, updates) + + # Also write to pixels.population column for population data + if metadata_field_name == 'population': + pop_updates = [ + (json.loads(metadata_json)[metadata_field_name], quadkey) + for quadkey, metadata_json in updates + ] + cursor.executemany(""" + UPDATE pixels SET population = %s WHERE quadkey = %s + """, pop_updates) + conn.commit() total_updated += len(updates) @@ -354,11 +365,11 @@ async def enrich_area_pixels( SET cached_population = sub.total_pop FROM ( SELECT pa.campaign_area_id, - COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_pop + COALESCE(SUM(p.population), 0) as total_pop FROM pixel_area pa - JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey + JOIN pixels p ON pa.quadkey = p.quadkey JOIN campaign_areas ca2 ON pa.campaign_area_id = ca2.id - WHERE ca2.campaign_id = %s AND pm.metadata ? 'population' + WHERE ca2.campaign_id = %s GROUP BY pa.campaign_area_id ) sub WHERE ca.id = sub.campaign_area_id diff --git a/truecover-backend/temporal/activities/entity_export.py b/truecover-backend/temporal/activities/entity_export.py index ecaa31645..0121020a4 100644 --- a/truecover-backend/temporal/activities/entity_export.py +++ b/truecover-backend/temporal/activities/entity_export.py @@ -1,5 +1,5 @@ -# ABOUTME: Temporal activities for exporting pixel coverage data to ODK entity lists -# ABOUTME: Handles fetching pixel data and creating entities via Ona API +# ABOUTME: Temporal activities for exporting coverage data to ODK entity lists +# ABOUTME: Handles fetching pixel and location data and creating entities via Ona API from temporalio import activity from typing import List, Dict, Any @@ -85,7 +85,7 @@ async def fetch_pixel_coverage_activity( p.adm4_pcode, pc.rounds FROM coverage_pixel pc - JOIN pixels p ON p.quadkey = pc.quadkey AND p.campaign_id = pc.campaign_id + JOIN pixels p ON p.quadkey = pc.quadkey WHERE pc.campaign_id = %s AND pc.indicator_id = %s AND pc.rounds && %s::integer[] @@ -111,19 +111,84 @@ async def fetch_pixel_coverage_activity( return_db_connection(conn) +@activity.defn +async def fetch_location_coverage_activity( + campaign_id: str, + indicator_id: str, + round_ids: List[str] +) -> List[Dict[str, Any]]: + """ + Fetch location coverage data filtered by selected rounds. + + Returns list of location coverage records with location details. + """ + conn = get_db_connection() + cursor = conn.cursor() + + try: + # Get round numbers from round IDs + placeholders = ','.join(['%s'] * len(round_ids)) + cursor.execute(f""" + SELECT round_number + FROM rounds + WHERE id IN ({placeholders}) + """, tuple(round_ids)) + + round_numbers = [row[0] for row in cursor.fetchall()] + + if not round_numbers: + return [] + + cursor.execute(""" + SELECT DISTINCT + c.id, + l.external_id, + l.latitude, + l.longitude, + l.quadkey, + c.rounds + FROM coverage c + JOIN locations l ON l.id = c.location_id + WHERE c.campaign_id = %s + AND c.indicator_id = %s + AND c.rounds && %s::integer[] + ORDER BY l.external_id + """, (campaign_id, indicator_id, round_numbers)) + + locations = [] + for row in cursor.fetchall(): + cov_id, external_id, latitude, longitude, quadkey, rounds = row + locations.append({ + 'id': str(cov_id), + 'external_id': external_id or '', + 'latitude': float(latitude) if latitude else None, + 'longitude': float(longitude) if longitude else None, + 'quadkey': quadkey or '', + 'rounds': rounds or [] + }) + + return locations + + finally: + cursor.close() + return_db_connection(conn) + + @activity.defn async def create_odk_entity_activity( project_id: str, - pixel_data: Dict[str, Any], - geometry_type: str = 'centroid' + entity_data: Dict[str, Any], + geometry_type: str = 'centroid', + entity_type: str = 'pixel' ) -> Dict[str, Any]: """ Create a single ODK entity via Ona API. Args: project_id: Project ID to get ODK credentials - pixel_data: Pixel data dict with id, quadkey, lat, lng, adm4_pcode + entity_data: Entity data dict (pixel or location fields) geometry_type: 'centroid' or 'boundary' - how to represent pixel geometry + entity_type: 'pixel' or 'location' Returns: Dict with success status and created entity info @@ -154,21 +219,27 @@ async def create_odk_entity_activity( # Remove trailing slash from host_url host_url = host_url.rstrip('/') - # Format geometry based on project setting - if geometry_type == 'boundary': - # Use pixel boundary (geoshape polygon) - geometry = get_pixel_boundary_coords(pixel_data['quadkey']) + if entity_type == 'location': + # Locations always use point geometry + label = entity_data['external_id'] + geometry = f"{entity_data['latitude']} {entity_data['longitude']} 0 0" + details = entity_data['quadkey'] else: - # Use pixel centroid (geopoint) - geometry = f"{pixel_data['latitude']} {pixel_data['longitude']} 0 0" + # Pixels use configurable geometry + label = entity_data['quadkey'] + if geometry_type == 'boundary': + geometry = get_pixel_boundary_coords(entity_data['quadkey']) + else: + geometry = f"{entity_data['latitude']} {entity_data['longitude']} 0 0" + details = entity_data['adm4_pcode'] # Build entity payload entity_payload = { - 'label': pixel_data['quadkey'], + 'label': label, 'data': { 'geometry': geometry, 'status': 'not_visited', - 'details': pixel_data['adm4_pcode'] + 'details': details } } @@ -199,7 +270,7 @@ async def create_odk_entity_activity( return { 'success': True, - 'quadkey': pixel_data['quadkey'], + 'label': label, 'entity_uuid': entity_result.get('uuid') } diff --git a/truecover-backend/temporal/activities/rounds.py b/truecover-backend/temporal/activities/rounds.py index 85da07533..0fd225020 100644 --- a/truecover-backend/temporal/activities/rounds.py +++ b/truecover-backend/temporal/activities/rounds.py @@ -116,12 +116,8 @@ async def fetch_coverage_for_sampling( pop_filter = "" pop_params = [] - if min_population is not None and population_field: - import re - if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', population_field): - raise ValueError(f"Invalid population field name: {population_field}") - pop_join = "LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey" - pop_filter = f"AND (pm.metadata->>'{population_field}')::float >= %s" + if min_population is not None: + pop_filter = "AND p.population >= %s" pop_params = [min_population] if allow_revisit: diff --git a/truecover-backend/temporal/workflows/entity_export.py b/truecover-backend/temporal/workflows/entity_export.py index 66d760c3f..621ac77c9 100644 --- a/truecover-backend/temporal/workflows/entity_export.py +++ b/truecover-backend/temporal/workflows/entity_export.py @@ -1,5 +1,5 @@ -# ABOUTME: Temporal workflow for exporting pixel coverage data to ODK entity lists -# ABOUTME: Orchestrates fetching pixel data and creating entities via Ona API +# ABOUTME: Temporal workflow for exporting coverage data to ODK entity lists +# ABOUTME: Orchestrates fetching pixel/location data and creating entities via Ona API from datetime import timedelta from typing import Any, Dict, List @@ -10,6 +10,7 @@ with workflow.unsafe.imports_passed_through(): from ..activities.entity_export import ( fetch_pixel_coverage_activity, + fetch_location_coverage_activity, create_odk_entity_activity ) @@ -17,17 +18,17 @@ @workflow.defn class EntityExportWorkflow: """ - Workflow for exporting pixel coverage data to ODK entity lists. + Workflow for exporting coverage data (pixels and locations) to ODK entity lists. Steps: - 1. Fetch pixel coverage data filtered by selected rounds - 2. Create ODK entity for each pixel (stops on first error) + 1. Fetch pixel and/or location coverage data filtered by selected rounds + 2. Create ODK entity for each item (stops on first error) """ def __init__(self): - self.total_pixels = 0 - self.created_pixels = 0 - self.current_quadkey = "" + self.total_entities = 0 + self.created_entities = 0 + self.current_label = "" self.error_message = None @workflow.run @@ -35,90 +36,96 @@ async def run( self, campaign_id: str, indicator_id: str, - round_ids: List[str], + pixel_round_ids: List[str], + location_round_ids: List[str], project_id: str, geometry_type: str = 'centroid' ) -> Dict[str, Any]: - """ - Run entity export workflow. - - Args: - campaign_id: Area ID - indicator_id: Indicator ID - round_ids: List of round IDs to filter by - project_id: Project ID for ODK credentials - - Returns: - Result summary with counts - - Raises: - Exception if any entity creation fails - """ - workflow.logger.info(f"Starting entity export for area {campaign_id}") - - # Fetch pixel coverage data - pixels = await workflow.execute_activity( - fetch_pixel_coverage_activity, - args=[campaign_id, indicator_id, round_ids], - start_to_close_timeout=timedelta(minutes=2), - retry_policy=RetryPolicy(maximum_attempts=3) - ) - - self.total_pixels = len(pixels) - workflow.logger.info(f"Found {self.total_pixels} pixels to export") - - if self.total_pixels == 0: + workflow.logger.info(f"Starting entity export for campaign {campaign_id}") + + entities = [] + + # Fetch pixel coverage data if pixel rounds selected + if pixel_round_ids: + pixels = await workflow.execute_activity( + fetch_pixel_coverage_activity, + args=[campaign_id, indicator_id, pixel_round_ids], + start_to_close_timeout=timedelta(minutes=2), + retry_policy=RetryPolicy(maximum_attempts=3) + ) + for p in pixels: + p['entity_type'] = 'pixel' + entities.extend(pixels) + workflow.logger.info(f"Found {len(pixels)} pixels to export") + + # Fetch location coverage data if location rounds selected + if location_round_ids: + locations = await workflow.execute_activity( + fetch_location_coverage_activity, + args=[campaign_id, indicator_id, location_round_ids], + start_to_close_timeout=timedelta(minutes=2), + retry_policy=RetryPolicy(maximum_attempts=3) + ) + for loc in locations: + loc['entity_type'] = 'location' + entities.extend(locations) + workflow.logger.info(f"Found {len(locations)} locations to export") + + self.total_entities = len(entities) + + if self.total_entities == 0: return { "success": True, - "total_pixels": 0, - "created_pixels": 0, - "message": "No pixels found for selected rounds" + "total_entities": 0, + "created_entities": 0, + "message": "No entities found for selected rounds" } - # Create ODK entity for each pixel - # Stop immediately on first error (no retry policy) - for pixel in pixels: - self.current_quadkey = pixel['quadkey'] - workflow.logger.info(f"Creating entity for pixel {self.current_quadkey}") + # Create ODK entity for each item + # Stop immediately on first error + for entity in entities: + entity_type = entity['entity_type'] + label = entity.get('external_id') if entity_type == 'location' else entity.get('quadkey') + self.current_label = label or '' + workflow.logger.info(f"Creating {entity_type} entity: {self.current_label}") try: result = await workflow.execute_activity( create_odk_entity_activity, - args=[project_id, pixel, geometry_type], + args=[project_id, entity, geometry_type, entity_type], start_to_close_timeout=timedelta(seconds=30), retry_policy=RetryPolicy(maximum_attempts=1) ) - self.created_pixels += 1 + self.created_entities += 1 workflow.logger.info( - f"Created entity {self.created_pixels}/{self.total_pixels}: " - f"{result['quadkey']}" + f"Created entity {self.created_entities}/{self.total_entities}: " + f"{result['label']}" ) except Exception as e: - # Stop on first error - error_msg = f"Failed to create entity for pixel {self.current_quadkey}: {str(e)}" + error_msg = f"Failed to create {entity_type} entity {self.current_label}: {str(e)}" self.error_message = error_msg workflow.logger.error(error_msg) raise Exception(error_msg) workflow.logger.info( - f"Entity export complete: {self.created_pixels} entities created" + f"Entity export complete: {self.created_entities} entities created" ) return { "success": True, - "total_pixels": self.total_pixels, - "created_pixels": self.created_pixels, - "message": f"Successfully created {self.created_pixels} entities in ODK" + "total_entities": self.total_entities, + "created_entities": self.created_entities, + "message": f"Successfully created {self.created_entities} entities in ODK" } @workflow.query def get_progress(self) -> Dict[str, Any]: """Query to get current workflow progress.""" return { - "total_pixels": self.total_pixels, - "created_pixels": self.created_pixels, - "current_quadkey": self.current_quadkey, + "total_entities": self.total_entities, + "created_entities": self.created_entities, + "current_label": self.current_label, "error_message": self.error_message }