diff --git a/docs/plans/replacement.md b/docs/plans/replacement.md new file mode 100644 index 000000000..c01141ba6 --- /dev/null +++ b/docs/plans/replacement.md @@ -0,0 +1 @@ +Okay, we want to add an additional replacement pixel. So for every time we select a pixel, we want to take one pixel that falls within the ring of the quad key. So basically, you know, the one right above, below, to the right, to the left, or to like top right, you know, northeast, southeast, etc. So kind of like the nine, sorry, one, two, three, four, five, six, seven. The eight pixels that go around a pixel are candidates. Then you would randomly select one of those. The only criteria would be that that that has the necessary amount of buildings. So five buildings or whatever the threshold is, you pick one of those, and then we're going to make that a replacement sample in the coverage database. And then, so we have to mark that separately, and then, second, we will make it blue so it's visible as a different color option. And then the idea is they can then pick that as a replacement if the primary one is uh not does it provide enough children. diff --git a/truecover-app/src/components/AddCampaignAreaModal.tsx b/truecover-app/src/components/AddCampaignAreaModal.tsx index dce2cae22..80daaac6b 100644 --- a/truecover-app/src/components/AddCampaignAreaModal.tsx +++ b/truecover-app/src/components/AddCampaignAreaModal.tsx @@ -19,6 +19,7 @@ interface AddCampaignAreaModalProps { name: string; } | null; drawnGeometry?: any; + existingBuildingCount?: number; onAreaAdded?: (result?: { areaId: string; buildingWorkflowId?: string }) => void; } @@ -29,6 +30,7 @@ const AddCampaignAreaModal: React.FC = ({ mode, adminBoundary, drawnGeometry, + existingBuildingCount = 0, onAreaAdded }) => { const { getToken } = useAuth(); @@ -177,6 +179,14 @@ const AddCampaignAreaModal: React.FC = ({

+ {existingBuildingCount > 0 && ( +
+ + This area already has {existingBuildingCount.toLocaleString()} buildings imported. + +
+ )} +
= ({ className="w-4 h-4" />

- Import building footprints from Overture Maps for this area + {existingBuildingCount > 0 + ? 'This will re-import building footprints (duplicates are skipped)' + : 'Import building footprints from Overture Maps for this area'}

diff --git a/truecover-app/src/components/CampaignAreasManager.tsx b/truecover-app/src/components/CampaignAreasManager.tsx index d06210c59..210d9f96b 100644 --- a/truecover-app/src/components/CampaignAreasManager.tsx +++ b/truecover-app/src/components/CampaignAreasManager.tsx @@ -68,6 +68,7 @@ const CampaignAreasManager: React.FC = ({ const [isDeleting, setIsDeleting] = useState(false); const [areaToEdit, setAreaToEdit] = useState(null); const [sampleCount, setSampleCount] = useState(50); + const [buildingsPerPixel, setBuildingsPerPixel] = useState(5); const [selectedRoundId, setSelectedRoundId] = useState(''); const [sampleTarget, setSampleTarget] = useState('pixels'); const [isSampling, setIsSampling] = useState(false); @@ -296,7 +297,8 @@ const CampaignAreasManager: React.FC = ({ sample_count: sampleCount, resample, round_id: selectedRoundId, - sample_target: sampleTarget + sample_target: sampleTarget, + ...(sampleTarget === 'pixels' && buildingsPerPixel > 0 ? { buildings_per_pixel: buildingsPerPixel } : {}), }, token ); @@ -765,6 +767,25 @@ const CampaignAreasManager: React.FC = ({ className="w-full px-3 py-2 bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-text-primary font-mono text-sm focus:border-tactical-accent-primary focus:outline-none" />
+ + {/* Buildings per Pixel (only for pixel sampling) */} + {sampleTarget === 'pixels' && ( +
+ + setBuildingsPerPixel(Math.max(0, parseInt(e.target.value) || 0))} + min={0} + className="w-full px-3 py-2 bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-text-primary font-mono text-sm focus:border-tactical-accent-primary focus:outline-none" + /> +

+ {buildingsPerPixel === 0 ? 'No building sampling' : `Up to ${buildingsPerPixel} buildings selected per sampled pixel`} +

+
+ )} )} diff --git a/truecover-app/src/components/CreateRoundModal.tsx b/truecover-app/src/components/CreateRoundModal.tsx index 109625b08..1dace85dd 100644 --- a/truecover-app/src/components/CreateRoundModal.tsx +++ b/truecover-app/src/components/CreateRoundModal.tsx @@ -1,9 +1,12 @@ +// ABOUTME: Modal for creating a sampling round with optional per-area auto-sampling +// ABOUTME: Creates a round record and optionally starts CampaignAreaSamplingWorkflow per selected area + import React, { useState, useEffect } from 'react'; import { TacticalModal, TacticalInput, TacticalButton, TacticalTextarea, TacticalSelect, TacticalDatePicker, tacticalToast } from '../tactical-ui'; import axios from 'axios'; import { useAuth } from '@clerk/clerk-react'; import { useIndicators } from '../hooks/useIndicators'; -import { usePixelMetadataStats } from '../hooks/usePixels'; +import { CampaignArea } from '../hooks/useCampaignAreas'; import { env } from '../config/env'; const API_URL = env.VITE_API_URL; @@ -14,8 +17,8 @@ interface CreateRoundModalProps { campaignId: string; projectId: string; onRoundCreated: () => void; - adminBoundaryPcode?: string; - adminBoundaryName?: string; + campaignAreas?: CampaignArea[]; + onSamplingStarted?: (areaWorkflowMap: Record) => void; } const CreateRoundModal: React.FC = ({ @@ -24,42 +27,21 @@ const CreateRoundModal: React.FC = ({ campaignId, projectId, onRoundCreated, - adminBoundaryPcode, - adminBoundaryName, + campaignAreas = [], + onSamplingStarted, }) => { const { getToken } = useAuth(); const { data: indicators } = useIndicators(projectId); - const { data: metadataStats } = usePixelMetadataStats(campaignId); const [name, setName] = useState(''); const [description, setDescription] = useState(''); const [startDate, setStartDate] = useState(''); const [endDate, setEndDate] = useState(''); const [selectedIndicatorId, setSelectedIndicatorId] = useState(''); - const [batchSize, setBatchSize] = useState('10'); - const [uncertaintyField, setUncertaintyField] = useState('prevalence_bci_width'); - const [allowRevisit, setAllowRevisit] = useState(false); - const [samplingTarget, setSamplingTarget] = useState<'locations' | 'pixels'>('locations'); const [isSubmitting, setIsSubmitting] = useState(false); const [error, setError] = useState(null); - const [enablePopulationFilter, setEnablePopulationFilter] = useState(false); - const [minPopulation, setMinPopulation] = useState('10'); - const [populationField, setPopulationField] = useState(''); - - // Get numeric metadata fields that could be used for population filtering - const numericMetadataFields = (metadataStats?.metadata_fields || []).filter( - (field: any) => field.data_type === 'integer' || field.data_type === 'float' - ); - // Auto-select population field if it exists, otherwise first numeric field - useEffect(() => { - if (numericMetadataFields.length > 0 && !populationField) { - // Prefer a field named "population" (case-insensitive) - const populationFieldMatch = numericMetadataFields.find( - (field: any) => field.name.toLowerCase() === 'population' - ); - setPopulationField(populationFieldMatch?.name || numericMetadataFields[0].name); - } - }, [numericMetadataFields, populationField]); + // Per-area selection: area_id -> sampling config + const [selectedAreas, setSelectedAreas] = useState>(new Map()); // Auto-select first indicator when indicators load useEffect(() => { @@ -68,6 +50,40 @@ const CreateRoundModal: React.FC = ({ } }, [indicators]); + const toggleArea = (areaId: string) => { + setSelectedAreas(prev => { + const next = new Map(prev); + if (next.has(areaId)) { + next.delete(areaId); + } else { + next.set(areaId, { sampleCount: 50, buildingsPerPixel: 5 }); + } + return next; + }); + }; + + const updateSampleCount = (areaId: string, count: number) => { + setSelectedAreas(prev => { + const next = new Map(prev); + const current = next.get(areaId); + if (current) { + next.set(areaId, { ...current, sampleCount: count }); + } + return next; + }); + }; + + const updateBuildingsPerPixel = (areaId: string, count: number) => { + setSelectedAreas(prev => { + const next = new Map(prev); + const current = next.get(areaId); + if (current) { + next.set(areaId, { ...current, buildingsPerPixel: count }); + } + return next; + }); + }; + const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); setError(null); @@ -82,17 +98,19 @@ const CreateRoundModal: React.FC = ({ return; } - const batchSizeNum = parseInt(batchSize); - if (isNaN(batchSizeNum) || batchSizeNum < 1) { - setError('Batch size must be a number greater than 0'); - return; - } - setIsSubmitting(true); try { const token = await getToken(); + // Build sample_areas array from checked areas + const sampleAreas = Array.from(selectedAreas.entries()).map(([areaId, config]) => ({ + area_id: areaId, + sample_count: config.sampleCount, + sample_target: 'pixels', + buildings_per_pixel: config.buildingsPerPixel, + })); + const response = await axios.post( `${API_URL}/api/campaigns/${campaignId}/rounds`, { @@ -101,17 +119,7 @@ const CreateRoundModal: React.FC = ({ start_date: startDate || null, end_date: endDate || null, indicator_id: selectedIndicatorId, - batch_size: batchSizeNum, - uncertainty_field: uncertaintyField, - allow_revisit: allowRevisit, - sampling_target: samplingTarget, - admin_pcode: adminBoundaryPcode || null, - min_population: (samplingTarget === 'pixels' && enablePopulationFilter && populationField) - ? parseFloat(minPopulation) - : null, - population_field: (samplingTarget === 'pixels' && enablePopulationFilter && populationField) - ? populationField - : null, + ...(sampleAreas.length > 0 ? { sample_areas: sampleAreas } : {}), }, { headers: { @@ -121,9 +129,8 @@ const CreateRoundModal: React.FC = ({ } ); - // Temporal workflow started - close modal immediately - if (response.status === 202 && response.data.workflow_id) { - const workflowId = response.data.workflow_id; + if (response.status === 201) { + const workflowIds = response.data.workflow_ids || {}; const roundName = name.trim(); // Reset form @@ -132,58 +139,25 @@ const CreateRoundModal: React.FC = ({ setStartDate(''); setEndDate(''); setSelectedIndicatorId(indicators && indicators.length > 0 ? indicators[0].id : ''); - setBatchSize('10'); - setUncertaintyField('prevalence_bci_width'); - setAllowRevisit(false); - setSamplingTarget('locations'); - setEnablePopulationFilter(false); - setMinPopulation('10'); - setPopulationField(numericMetadataFields.length > 0 ? numericMetadataFields[0].name : ''); - - // Close modal immediately - onClose(); + setSelectedAreas(new Map()); - // Show info toast that round generation started - tacticalToast.info( - 'Round Generation Started', - `Generating round "${roundName}"...` - ); - - // Poll for completion - const pollInterval = setInterval(async () => { - try { - const statusResponse = await axios.get( - `${API_URL}/api/rounds/workflow/${workflowId}/status`, - { - headers: { - Authorization: `Bearer ${token}`, - }, - } - ); - - if (statusResponse.data.status === 'completed' && statusResponse.data.result) { - clearInterval(pollInterval); - const selectedCount = statusResponse.data.result.selected_count || 0; - tacticalToast.success( - 'Round Generated', - `Created round "${roundName}" with ${selectedCount.toLocaleString()} ${samplingTarget} selected` - ); - // Trigger parent refresh - onRoundCreated(); - } else if (statusResponse.data.status === 'failed') { - clearInterval(pollInterval); - tacticalToast.error( - 'Round Generation Failed', - statusResponse.data.error || 'Round generation failed' - ); - } - } catch (err) { - console.error('Failed to check workflow status:', err); + onClose(); + onRoundCreated(); + + if (Object.keys(workflowIds).length > 0) { + tacticalToast.info( + 'Sampling Started', + `Created round "${roundName}" and started sampling for ${Object.keys(workflowIds).length} area(s)` + ); + if (onSamplingStarted) { + onSamplingStarted(workflowIds); } - }, 2000); - - // Clean up interval after 10 minutes (failsafe) - setTimeout(() => clearInterval(pollInterval), 600000); + } else { + tacticalToast.success( + 'Round Created', + `Created round "${roundName}"` + ); + } } } catch (err: any) { console.error('Error creating round:', err); @@ -204,6 +178,18 @@ const CreateRoundModal: React.FC = ({ } }; + const areaDisplayName = (area: CampaignArea) => { + const parts = [ + area.name || area.admin_boundary_name || 'Unnamed Area', + ]; + const hierarchy = [area.division_name, area.district_name, area.upazila_name, area.union_name] + .filter(Boolean); + if (hierarchy.length > 0) { + parts.push(`(${hierarchy.join(' > ')})`); + } + return parts.join(' '); + }; + return ( = ({ >
- {/* Admin Boundary Filter Indicator */} - {adminBoundaryPcode && adminBoundaryName && ( -
-

- Filtering to Admin Boundary -

-

- {adminBoundaryName} ({adminBoundaryPcode}) -

-

- This round will only include {samplingTarget === 'locations' ? 'locations' : 'pixels'} within this administrative boundary. -

-
- )} - = ({ value={description} onChange={setDescription} placeholder="Describe the purpose of this data collection round..." - rows={3} + rows={2} disabled={isSubmitting} /> @@ -259,17 +230,6 @@ const CreateRoundModal: React.FC = ({ disabled={isSubmitting} /> - setSamplingTarget(value as 'locations' | 'pixels')} - options={[ - { value: 'locations', label: 'Locations' }, - { value: 'pixels', label: 'Pixels' }, - ]} - disabled={isSubmitting} - /> -
= ({ />
-
-

- Adaptive Sampling Parameters -

- -
- - - - -
- setAllowRevisit(e.target.checked)} - disabled={isSubmitting} - className="w-4 h-4 bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-accent-orange focus:ring-tactical-accent-orange focus:ring-2" - /> - -
+ {/* Campaign Areas Selection */} + {campaignAreas.length > 0 && ( +
+

+ Adaptively Sample Areas +

+

+ Select areas to adaptively sample when this round is created. Leave unchecked to sample manually later. +

- {/* Population Filter - Only show for pixels when metadata fields are available */} - {samplingTarget === 'pixels' && numericMetadataFields.length > 0 && ( -
-
- setEnablePopulationFilter(e.target.checked)} - disabled={isSubmitting} - className="w-4 h-4 bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-accent-orange focus:ring-tactical-accent-orange focus:ring-2" - /> - -
- - {enablePopulationFilter && ( -
- ({ - value: field.name, - label: `${field.name}${field.unit ? ` (${field.unit})` : ''}` - }))} - disabled={isSubmitting} - /> - - - -

- Only pixels with {populationField || 'population'} ≥ {minPopulation || '10'} will be included in sampling. -

-
- )} -
- )} +
+ + + + + + + + + + + + + {campaignAreas.map((area) => { + const isSelected = selectedAreas.has(area.id); + return ( + + + + + + + + + ); + })} + +
AreaPixelsPopulationSample CountBldgs/Pixel
+ toggleArea(area.id)} + disabled={isSubmitting} + className="w-4 h-4 bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-accent-orange focus:ring-tactical-accent-orange focus:ring-2" + /> + toggleArea(area.id)} + > + {areaDisplayName(area)} + + {area.pixel_count.toLocaleString()} + + {area.total_population.toLocaleString()} + + {isSelected ? ( + updateSampleCount(area.id, parseInt(e.target.value) || 1)} + disabled={isSubmitting} + className="w-24 px-2 py-1 text-sm font-mono text-right bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-text-primary focus:border-tactical-accent-orange focus:outline-none" + /> + ) : ( + — + )} + + {isSelected ? ( + updateBuildingsPerPixel(area.id, parseInt(e.target.value) || 0)} + disabled={isSubmitting} + className="w-24 px-2 py-1 text-sm font-mono text-right bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-text-primary focus:border-tactical-accent-orange focus:outline-none" + /> + ) : ( + — + )} +
+
-
+ )}
{error && ( @@ -405,6 +361,8 @@ const CreateRoundModal: React.FC = ({ CREATING... + ) : selectedAreas.size > 0 ? ( + `Create Round & Sample ${selectedAreas.size} Area${selectedAreas.size > 1 ? 's' : ''}` ) : ( 'Create Round' )} diff --git a/truecover-app/src/components/MapView.tsx b/truecover-app/src/components/MapView.tsx index c74e6dc78..afbe0e881 100644 --- a/truecover-app/src/components/MapView.tsx +++ b/truecover-app/src/components/MapView.tsx @@ -51,6 +51,8 @@ interface MapViewProps { onToggleCampaignAreas?: () => void; onAddAdminBoundaryToCampaign?: (id: string, pcode: string, name: string, geometry?: any) => void; selectedAreaBounds?: [[number, number], [number, number]] | null; + savedViewState?: { longitude: number; latitude: number; zoom: number } | null; + onViewStateChange?: (viewState: { longitude: number; latitude: number; zoom: number }) => void; className?: string; } @@ -88,7 +90,7 @@ const getCentroid = (geometry: any): [number, number] => { const MARTIN_URL = env.VITE_MARTIN_URL; -const MapView: React.FC = ({ data, selectedData, locations, mode = 'sampling', highlightRounds = [], showSampled = true, onToggleSampled, interpolationMode = 'none', selectedMetadataField = '', metadataVisualizationMode = 'fill', showPixels = false, onTogglePixels, pixelsBounds, onBoundsChange, campaignId, indicatorId, pixelVersion, pixelCount = 0, onGeneratePixels, histogramBrushRanges = null, histogramDataType = 'locations', sampledItemsCount = 0, planningMode = false, campaignAreas = [], showCampaignAreas = true, onToggleCampaignAreas, onAddAdminBoundaryToCampaign, selectedAreaBounds, className }) => { +const MapView: React.FC = ({ data, selectedData, locations, mode = 'sampling', highlightRounds = [], showSampled = true, onToggleSampled, interpolationMode = 'none', selectedMetadataField = '', metadataVisualizationMode = 'fill', showPixels = false, onTogglePixels, pixelsBounds, onBoundsChange, campaignId, indicatorId, pixelVersion, pixelCount = 0, onGeneratePixels, histogramBrushRanges = null, histogramDataType = 'locations', sampledItemsCount = 0, planningMode = false, campaignAreas = [], showCampaignAreas = true, onToggleCampaignAreas, onAddAdminBoundaryToCampaign, selectedAreaBounds, savedViewState, onViewStateChange, className }) => { const [popupInfo, setPopupInfo] = useState(null); const [mapStyle, setMapStyle] = useState('mapbox://styles/mapbox/dark-v11'); const [viewportBounds, setViewportBounds] = useState<[[number, number], [number, number]] | null>(null); @@ -660,9 +662,8 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = type: 'fill', filter: ['in', ['geometry-type'], ['literal', ['Polygon', 'MultiPolygon']]], paint: { - // When both visit locations AND interpolation are active, make fill transparent - 'fill-color': (showSampled && interpolationMode !== 'none') ? 'rgba(40, 167, 69, 0)' : '#28a745', - 'fill-opacity': (showSampled && interpolationMode !== 'none') ? 0 : 0.95 + 'fill-color': 'rgba(40, 167, 69, 0)', + 'fill-opacity': 0 } }; @@ -672,7 +673,7 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = filter: ['in', ['geometry-type'], ['literal', ['Polygon', 'MultiPolygon']]], paint: { 'line-color': '#28a745', - 'line-width': 2, + 'line-width': 4, 'line-opacity': 1 } }; @@ -825,6 +826,12 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = onBoundsChange([sw.lng, sw.lat, ne.lng, ne.lat]); } } + + // Report current view state to parent for persistence across remounts + if (onViewStateChange) { + const center = map.getCenter(); + onViewStateChange({ longitude: center.lng, latitude: center.lat, zoom }); + } }; const handleConfirmPixelGeneration = async () => { @@ -1029,7 +1036,7 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = 'case', shouldShowLocation(), '#28a745', - 'rgba(153, 153, 153, 0)' + '#ffffff' ]; } return 'rgba(153, 153, 153, 0)'; @@ -1052,11 +1059,11 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = return [ 'case', shouldShowLocation(), - 0.95, - 0 + 0.7, + 0.1 ]; } - return 0; + return 0.1; }; const getPolygonLineColor = () => { @@ -1152,7 +1159,7 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = 'case', shouldShowLocation(), '#28a745', - 'rgba(255, 255, 255, 0)' + '#ffffff' ]; } return 'rgba(255, 255, 255, 0)'; @@ -1189,7 +1196,7 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = 'case', shouldShowLocation(), 1, - 0 + 0.9 ]; } return 0.8; @@ -1201,7 +1208,11 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = = ({ data, selectedData, locations, mode = 0.9, 0 ] - : 0.6 + : showSampled + ? [ + 'interpolate', ['linear'], ['zoom'], + 14, ['case', shouldShowLocation(), 0.6, 0.6], + 15, ['case', shouldShowLocation(), 0, 0.6] + ] + : 0.6 }} /> = ({ data, selectedData, locations, mode = : mapStyle === 'mapbox://styles/mapbox/satellite-streets-v12' ? '#ffffff' : '#28a745', - 'line-width': 1, + 'line-width': showSampled + ? [ + 'interpolate', ['linear'], ['zoom'], + 14, ['case', shouldShowLocation(), 1, 1], + 15, ['case', shouldShowLocation(), 4, 1] + ] + : 1, 'line-opacity': interpolationMode === 'coverage' ? [ 'case', @@ -1582,7 +1605,14 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = 0.3, 0 ] - : 0.6 + : showSampled + ? [ + 'case', + shouldShowLocation(), + 1, + 0.6 + ] + : 0.6 }} /> @@ -1646,6 +1676,66 @@ const MapView: React.FC = ({ data, selectedData, locations, mode = }} /> )} + + {/* Pixel quadkey label: anchored at top-left corner point */} + + {/* Pixel stats label: anchored at bottom-right corner point */} + )} diff --git a/truecover-app/src/components/PredictedCoverageSection.tsx b/truecover-app/src/components/PredictedCoverageSection.tsx index b5a134a03..22a69a068 100644 --- a/truecover-app/src/components/PredictedCoverageSection.tsx +++ b/truecover-app/src/components/PredictedCoverageSection.tsx @@ -196,6 +196,7 @@ const PredictedCoverageSection: React.FC = ({ Coverage ID Quadkey + Sampled/Pixel Rounds Latitude Longitude @@ -208,7 +209,15 @@ const PredictedCoverageSection: React.FC = ({ - {coverageData.map((record) => { + {(() => { + // Count sampled buildings per pixel (quadkey) + const sampledPerPixel = new Map(); + for (const r of coverageData) { + if (r.rounds && r.rounds.length > 0 && r.quadkey) { + sampledPerPixel.set(r.quadkey, (sampledPerPixel.get(r.quadkey) || 0) + 1); + } + } + return coverageData.map((record) => { const hasRounds = record.rounds && record.rounds.length > 0; const rowTextColor = hasRounds ? 'text-tactical-accent-green' : ''; return ( @@ -222,6 +231,9 @@ const PredictedCoverageSection: React.FC = ({ {record.quadkey || '-'} + + {record.quadkey ? (sampledPerPixel.get(record.quadkey) || 0) : '-'} + {hasRounds ? record.rounds.sort((a, b) => a - b).map((roundNum, idx) => ( @@ -272,7 +284,8 @@ const PredictedCoverageSection: React.FC = ({ ); - })} + }); + })()} {isLoadingMore && ( diff --git a/truecover-app/src/components/RoundsManager.tsx b/truecover-app/src/components/RoundsManager.tsx index efa032e66..03c71280a 100644 --- a/truecover-app/src/components/RoundsManager.tsx +++ b/truecover-app/src/components/RoundsManager.tsx @@ -6,6 +6,7 @@ import { StratifiedClusterSamplingWizard } from './StratifiedClusterSamplingWiza import axios from 'axios'; import { useAuth } from '@clerk/clerk-react'; import { useIndicators } from '../hooks/useIndicators'; +import { CampaignArea } from '../hooks/useCampaignAreas'; import { env } from '../config/env'; const API_URL = env.VITE_API_URL; @@ -32,12 +33,12 @@ interface RoundsManagerProps { onRoundSelected?: (roundNumber: number | null) => void; selectedAdminBoundary?: { pcode: string; name: string } | null; onClearAdminBoundary?: () => void; - pixelCount?: number; indicatorId?: string; onSamplingWorkflowsStarted?: (areaWorkflowMap: Record) => void; + campaignAreas?: CampaignArea[]; } -const RoundsManager: React.FC = ({ campaignId, areaName, projectId, locations, onRoundSelected, selectedAdminBoundary, onClearAdminBoundary, pixelCount = 0, indicatorId, onSamplingWorkflowsStarted }) => { +const RoundsManager: React.FC = ({ campaignId, areaName, projectId, locations, onRoundSelected, selectedAdminBoundary, onClearAdminBoundary, indicatorId, onSamplingWorkflowsStarted, campaignAreas }) => { const { getToken } = useAuth(); const { data: indicators } = useIndicators(projectId); const [rounds, setRounds] = useState([]); @@ -152,7 +153,6 @@ const RoundsManager: React.FC = ({ campaignId, areaName, pro variant="primary" size="sm" onClick={() => setIsCreateModalOpen(true)} - disabled={pixelCount === 0} > + Create New Round @@ -303,8 +303,8 @@ const RoundsManager: React.FC = ({ campaignId, areaName, pro campaignId={campaignId} projectId={projectId} onRoundCreated={handleRoundCreated} - adminBoundaryPcode={selectedAdminBoundary?.pcode} - adminBoundaryName={selectedAdminBoundary?.name} + campaignAreas={campaignAreas} + onSamplingStarted={onSamplingWorkflowsStarted} /> { // Indicator and Round filters const [selectedIndicatorId, setSelectedIndicatorId] = useState(''); - const [selectedRoundIds, setSelectedRoundIds] = useState<(string | number)[]>(['all']); + const [selectedRoundIds, setSelectedRoundIds] = useState<(string | number)[]>([]); + const [roundFilterInitialized, setRoundFilterInitialized] = useState(false); const [showSampled, setShowSampled] = useState(true); const [interpolationMode, setInterpolationMode] = useState<'none' | 'coverage' | 'uncertainty' | 'metadata'>('none'); const [selectedMetadataField, setSelectedMetadataField] = useState(''); @@ -75,6 +76,7 @@ const LocationsPage: React.FC = () => { const [currentMapBounds, setCurrentMapBounds] = useState<[number, number, number, number] | null>(null); const [planningMode, setPlanningMode] = useState(false); const [mapMode, setMapMode] = useState(false); + const [savedMapViewState, setSavedMapViewState] = useState<{ longitude: number; latitude: number; zoom: number } | null>(null); const [histogramDrawerOpen, setHistogramDrawerOpen] = useState(false); const [selectedAdminBoundary, setSelectedAdminBoundary] = useState<{ pcode: string; name: string } | null>(null); const [isAddCampaignAreaModalOpen, setIsAddCampaignAreaModalOpen] = useState(false); @@ -104,26 +106,20 @@ const LocationsPage: React.FC = () => { }, [selectedAreaId, campaignAreas]); // Compute roundId for coverage data query - // 'all' means "show only rows with ANY round assigned" - // specific round ID means "filter by that round" // empty array means "show everything" (no filter) + // 'sampled' means "show only rows with ANY round assigned" + // specific round ID means "filter by that round" const coverageRoundId = useMemo(() => { if (selectedRoundIds.length === 0) { - console.log('[Round Filter] Empty selection, no filter', selectedRoundIds); return undefined; } if (selectedRoundIds.includes('all')) { - // "All Rounds" = filter to show only rows with rounds assigned - console.log('[Round Filter] All rounds = show only sampled rows'); - return 'has_rounds'; // Special value to filter for non-empty rounds + return 'has_rounds'; } if (selectedRoundIds.length === 1) { - const roundId = String(selectedRoundIds[0]); - console.log('[Round Filter] Single round selected:', roundId); - return roundId; + return String(selectedRoundIds[0]); } - // Multiple specific rounds selected - for now treat as "has_rounds" - console.log('[Round Filter] Multiple rounds selected, showing all with rounds'); + // Multiple specific rounds selected return 'has_rounds'; }, [selectedRoundIds]); @@ -169,8 +165,8 @@ const LocationsPage: React.FC = () => { data: locationsResult, } = useInfiniteLocations(selectedCampaign?.id); - // Get total counts from first page (they're the same across all pages) - const locationTotalCount = locationsResult?.pages?.[0]?.total_count || locations?.total_count || locations?.locations?.length || 0; + // Get total counts from coverage data (round-filtered) when available, else from locations + const locationTotalCount = coverageDataResult?.pages?.[0]?.locationTotalCount ?? locationsResult?.pages?.[0]?.total_count ?? locations?.total_count ?? locations?.locations?.length ?? 0; const pixelTotalCount = coverageDataResult?.pages?.[0]?.pixelTotalCount || 0; // Set default indicator to first one when indicators load @@ -180,6 +176,14 @@ const LocationsPage: React.FC = () => { } }, [indicators]); + // Default round filter to "All Rounds" when rounds exist + useEffect(() => { + if (!roundFilterInitialized && rounds && rounds.length > 0) { + setSelectedRoundIds(['all']); + setRoundFilterInitialized(true); + } + }, [rounds, roundFilterInitialized]); + // Auto-enable pixels when they exist for the area useEffect(() => { if (pixelStats && pixelStats.count > 0) { @@ -467,6 +471,7 @@ const LocationsPage: React.FC = () => { })) ]} placeholder="Filter by Round" + autoApply />
@@ -595,25 +600,27 @@ const LocationsPage: React.FC = () => { setIsAddCampaignAreaModalOpen(true); }} selectedAreaBounds={selectedAreaBounds} + savedViewState={savedMapViewState} + onViewStateChange={setSavedMapViewState} className="h-full" /> - - {/* Histogram Drawer */} - {(interpolationMode === 'coverage' || interpolationMode === 'uncertainty' || interpolationMode === 'metadata') && ( - setHistogramDrawerOpen(!histogramDrawerOpen)} - histogramData={histogramData} - interpolationMode={interpolationMode} - indicatorName={indicators?.find(ind => ind.id === selectedIndicatorId)?.name} - onBrushChange={setHistogramBrushRanges} - histogramTab={histogramTab} - onTabChange={setHistogramTab} - locationTotalCount={locationTotalCount} - pixelTotalCount={pixelTotalCount} - /> - )} + + {/* Histogram Drawer */} + {(interpolationMode === 'coverage' || interpolationMode === 'uncertainty' || interpolationMode === 'metadata') && ( + setHistogramDrawerOpen(!histogramDrawerOpen)} + histogramData={histogramData} + interpolationMode={interpolationMode} + indicatorName={indicators?.find(ind => ind.id === selectedIndicatorId)?.name} + onBrushChange={setHistogramBrushRanges} + histogramTab={histogramTab} + onTabChange={setHistogramTab} + locationTotalCount={locationTotalCount} + pixelTotalCount={pixelTotalCount} + /> + )} ) : ( /* NORMAL LAYOUT */ @@ -789,6 +796,7 @@ const LocationsPage: React.FC = () => { })) ]} placeholder="Filter by Round" + autoApply /> @@ -932,6 +940,8 @@ const LocationsPage: React.FC = () => { setIsAddCampaignAreaModalOpen(true); }} selectedAreaBounds={selectedAreaBounds} + savedViewState={savedMapViewState} + onViewStateChange={setSavedMapViewState} /> @@ -1011,11 +1021,11 @@ const LocationsPage: React.FC = () => { onRoundSelected={setSelectedRoundFilter} selectedAdminBoundary={selectedAdminBoundary} onClearAdminBoundary={() => setSelectedAdminBoundary(null)} - pixelCount={pixelStats?.count || 0} indicatorId={selectedIndicatorId} onSamplingWorkflowsStarted={(areaWorkflowMap) => { setSamplingWorkflows(new Map(Object.entries(areaWorkflowMap))); }} + campaignAreas={campaignAreas || []} /> {/* Campaign Areas Section - Primary workflow entry point */} @@ -1145,6 +1155,11 @@ const LocationsPage: React.FC = () => { mode={campaignAreaMode} adminBoundary={selectedAdminBoundaryForCampaign} drawnGeometry={drawnGeometryForCampaign} + existingBuildingCount={ + campaignAreas?.find(a => + a.admin_boundary_name === selectedAdminBoundaryForCampaign?.name + )?.building_count || 0 + } onAreaAdded={(result) => { setCampaignAreasRefreshKey(prev => prev + 1); refetchPixelStats(); diff --git a/truecover-app/src/services/api.ts b/truecover-app/src/services/api.ts index b4d2f2fc0..6fb755d5b 100644 --- a/truecover-app/src/services/api.ts +++ b/truecover-app/src/services/api.ts @@ -332,6 +332,7 @@ export const campaignAreasApi = { resample?: boolean; round_id?: string; sample_target?: 'pixels' | 'buildings'; + buildings_per_pixel?: number; }, token: string ): Promise<{ workflow_id: string; area_id: string; status: string; message: string }> { diff --git a/truecover-app/src/tactical-ui/components/TacticalMultiSelect.tsx b/truecover-app/src/tactical-ui/components/TacticalMultiSelect.tsx index 13fa12f2b..b74b2e92b 100644 --- a/truecover-app/src/tactical-ui/components/TacticalMultiSelect.tsx +++ b/truecover-app/src/tactical-ui/components/TacticalMultiSelect.tsx @@ -175,8 +175,7 @@ export const TacticalMultiSelect: React.FC = ({ type="checkbox" checked={isOptionChecked(option.value)} onChange={() => {}} // Handled by div onClick - onClick={(e) => e.stopPropagation()} - className="w-4 h-4 bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-accent-orange focus:ring-tactical-accent-orange focus:ring-2" + className="w-4 h-4 bg-tactical-bg-tertiary border border-tactical-border-medium text-tactical-accent-orange focus:ring-tactical-accent-orange focus:ring-2 pointer-events-none" /> {option.label} diff --git a/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql b/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql index 66aa3497c..1e3b4e77b 100644 --- a/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql +++ b/truecover-backend/db/migrations/optimize_pixels_by_campaign.sql @@ -1,9 +1,11 @@ --- Optimize pixels_by_campaign: skip centroids query when metadata_field is not needed +-- ABOUTME: Tile function for pixels_by_campaign with polygon, label, and centroid layers. +-- ABOUTME: Labels (quadkey, population, building count) only generated at zoom 16+. CREATE OR REPLACE FUNCTION pixels_by_campaign(z integer, x integer, y integer, query_params json) RETURNS bytea AS $$ DECLARE mvt_polygons bytea; mvt_points bytea; + mvt_labels bytea; target_campaign_id uuid; target_indicator_id uuid; metadata_field text; @@ -52,6 +54,58 @@ BEGIN ) as tile WHERE geom IS NOT NULL; + -- Generate label points at pixel corners for zoom 16+ display. + -- Uses a CTE to scan the pixels table once, then generates both corner points. + IF z >= 16 THEN + SELECT INTO mvt_labels ST_AsMVT(tile, 'pixels_labels', 4096, 'geom') + FROM ( + WITH pixel_data AS ( + SELECT + p.quadkey, + p.geometry, + (pm.metadata->>'population')::numeric AS population, + lc.building_count + FROM pixels p + JOIN pixel_area pa ON p.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey + LEFT JOIN LATERAL ( + SELECT COUNT(*)::integer AS building_count + FROM locations l + WHERE l.quadkey = p.quadkey + ) lc ON true + WHERE ca.campaign_id = target_campaign_id + AND p.geometry && ST_Transform(ST_TileEnvelope(z, x, y), 4326) + ) + -- Top-left corner (quadkey label) + SELECT + ST_AsMVTGeom( + ST_Transform(ST_SetSRID(ST_MakePoint(ST_XMin(pd.geometry), ST_YMax(pd.geometry)), 4326), 3857), + ST_TileEnvelope(z, x, y), + 4096, 64, true + ) AS geom, + 'quadkey' AS label_type, + pd.quadkey, + NULL::numeric AS population, + NULL::integer AS building_count + FROM pixel_data pd + UNION ALL + -- Bottom-right corner (stats label) + SELECT + ST_AsMVTGeom( + ST_Transform(ST_SetSRID(ST_MakePoint(ST_XMax(pd.geometry), ST_YMin(pd.geometry)), 4326), 3857), + ST_TileEnvelope(z, x, y), + 4096, 64, true + ) AS geom, + 'stats' AS label_type, + pd.quadkey, + pd.population, + pd.building_count + FROM pixel_data pd + ) as tile + WHERE geom IS NOT NULL; + END IF; + -- Only generate centroids when metadata circle visualization is needed IF metadata_field IS NOT NULL AND metadata_field != '' THEN SELECT INTO mvt_points ST_AsMVT(tile, 'pixels_centroids', 4096, 'geom') @@ -79,6 +133,6 @@ BEGIN WHERE geom IS NOT NULL; END IF; - RETURN COALESCE(mvt_polygons, ''::bytea) || COALESCE(mvt_points, ''::bytea); + RETURN COALESCE(mvt_polygons, ''::bytea) || COALESCE(mvt_points, ''::bytea) || COALESCE(mvt_labels, ''::bytea); END $$ LANGUAGE plpgsql STABLE STRICT PARALLEL SAFE; diff --git a/truecover-backend/db/migrations_global_locations.py b/truecover-backend/db/migrations_global_locations.py new file mode 100644 index 000000000..1c668f7b6 --- /dev/null +++ b/truecover-backend/db/migrations_global_locations.py @@ -0,0 +1,212 @@ +# ABOUTME: Database migration to make locations global (not campaign-scoped) +# ABOUTME: Removes campaign_id from locations, updates constraints and tile functions + +from db.connection import get_db_connection, return_db_connection + + +def run_global_locations_migration(): + """ + Make locations global by removing campaign_id from the locations table. + + Locations associate with campaigns through spatial overlap with campaign areas + via quadkey/pixel_area joins. The coverage table retains campaign_id since + coverage predictions are campaign-specific. + + This migration: + 1. Updates coverage unique constraint to include campaign_id + 2. Adds global unique index on external_id + 3. Drops campaign_id from locations + 4. Recreates pixel_location_counts materialized view + 5. Recreates locations_by_campaign tile function + """ + conn = None + try: + conn = get_db_connection() + cursor = conn.cursor() + + print("=" * 60) + print("GLOBAL LOCATIONS MIGRATION") + print("=" * 60) + + # Step 1: Update coverage unique constraint + print("\n[Step 1] Updating coverage unique constraint...") + cursor.execute(""" + ALTER TABLE coverage DROP CONSTRAINT IF EXISTS coverage_location_id_indicator_id_version_key; + """) + cursor.execute(""" + ALTER TABLE coverage DROP CONSTRAINT IF EXISTS coverage_location_id_indicator_id_key; + """) + cursor.execute(""" + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'coverage_campaign_location_indicator_version_key' + ) THEN + ALTER TABLE coverage ADD CONSTRAINT coverage_campaign_location_indicator_version_key + UNIQUE(campaign_id, location_id, indicator_id, version); + END IF; + END $$; + """) + print(" - Updated coverage unique constraint to include campaign_id") + + # Step 2: Add global unique index on external_id + print("\n[Step 2] Adding global unique index on external_id...") + cursor.execute(""" + DROP INDEX IF EXISTS idx_locations_external_id; + """) + cursor.execute(""" + CREATE UNIQUE INDEX IF NOT EXISTS idx_locations_external_id_unique + ON locations(external_id) WHERE external_id IS NOT NULL; + """) + print(" - Created global unique index on external_id") + + # Step 3: Drop old materialized view (depends on campaign_id column) + print("\n[Step 3] Dropping old pixel_location_counts materialized view...") + cursor.execute(""" + DROP MATERIALIZED VIEW IF EXISTS pixel_location_counts CASCADE; + """) + print(" - Dropped pixel_location_counts") + + # Step 4: Drop campaign_id from locations + print("\n[Step 4] Dropping campaign_id from locations...") + cursor.execute(""" + ALTER TABLE locations DROP CONSTRAINT IF EXISTS locations_campaign_id_fkey; + """) + cursor.execute(""" + DROP INDEX IF EXISTS idx_locations_campaign_id; + """) + cursor.execute(""" + ALTER TABLE locations DROP COLUMN IF EXISTS campaign_id; + """) + print(" - Dropped campaign_id column from locations") + + # Step 5: Recreate pixel_location_counts materialized view (global) + print("\n[Step 5] Recreating pixel_location_counts materialized view...") + cursor.execute(""" + CREATE MATERIALIZED VIEW pixel_location_counts AS + SELECT quadkey, COUNT(*) as location_count + FROM locations WHERE quadkey IS NOT NULL + GROUP BY quadkey; + """) + cursor.execute(""" + CREATE UNIQUE INDEX idx_pixel_location_counts_quadkey + ON pixel_location_counts(quadkey); + """) + print(" - Recreated pixel_location_counts (global, no campaign_id)") + + cursor.execute("REFRESH MATERIALIZED VIEW pixel_location_counts") + print(" - Refreshed materialized view") + + # Step 6: Recreate locations_by_campaign tile function + print("\n[Step 6] Recreating locations_by_campaign tile function...") + cursor.execute("DROP FUNCTION IF EXISTS locations_by_campaign(integer, integer, integer, json) CASCADE;") + cursor.execute(""" + CREATE OR REPLACE FUNCTION locations_by_campaign(z integer, x integer, y integer, query_params json) + RETURNS bytea AS $$ + DECLARE + mvt bytea; + target_campaign_id uuid; + target_indicator_id uuid; + BEGIN + target_campaign_id := (query_params->>'campaign_id')::uuid; + target_indicator_id := NULLIF(query_params->>'indicator_id', '')::uuid; + + IF target_campaign_id IS NULL THEN + RETURN NULL; + END IF; + + SELECT INTO mvt ST_AsMVT(tile, 'locations', 4096, 'geom') + FROM ( + SELECT DISTINCT ON (l.id) + ST_AsMVTGeom( + CASE + WHEN z < 10 THEN ST_Simplify(ST_Transform(l.geometry, 3857), 100) + WHEN z < 12 THEN ST_Simplify(ST_Transform(l.geometry, 3857), 50) + WHEN z < 14 THEN ST_Simplify(ST_Transform(l.geometry, 3857), 20) + WHEN z < 16 THEN ST_Simplify(ST_Transform(l.geometry, 3857), 5) + ELSE ST_Transform(l.geometry, 3857) + END, + ST_TileEnvelope(z, x, y), + 4096, 256, true + ) AS geom, + l.id::text, + l.external_id, + l.latitude, + l.longitude, + l.properties, + c.rounds::text AS rounds, + c.n_trials, + c.n_covered, + c.prevalence_prediction, + c.prevalence_bci_width, + c.exceedance_probability, + c.exceedance_uncertainty + FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + LEFT JOIN LATERAL ( + SELECT + rounds, + n_trials, + n_covered, + prevalence_prediction, + prevalence_bci_width, + exceedance_probability, + exceedance_uncertainty + FROM coverage + WHERE location_id = l.id + AND campaign_id = target_campaign_id + AND (target_indicator_id IS NULL OR indicator_id = target_indicator_id) + ORDER BY version DESC + LIMIT 1 + ) c ON true + WHERE ca.campaign_id = target_campaign_id + AND l.geometry && ST_Transform(ST_TileEnvelope(z, x, y), 4326) + ) as tile + WHERE geom IS NOT NULL; + + RETURN mvt; + END + $$ LANGUAGE plpgsql STABLE STRICT PARALLEL SAFE; + """) + print(" - Recreated locations_by_campaign with spatial join") + + # Verify + print("\n[Verify] Checking schema...") + cursor.execute(""" + SELECT column_name FROM information_schema.columns + WHERE table_name = 'locations' AND column_name = 'campaign_id' + """) + if cursor.fetchone(): + raise RuntimeError("campaign_id still exists on locations table!") + + cursor.execute(""" + SELECT COUNT(*) FROM locations + """) + location_count = cursor.fetchone()[0] + print(f" - Locations count: {location_count:,}") + + conn.commit() + cursor.close() + + print("\n" + "=" * 60) + print("GLOBAL LOCATIONS MIGRATION COMPLETED SUCCESSFULLY") + print("=" * 60) + + return True + + except Exception as e: + if conn: + conn.rollback() + print(f"\nERROR: Migration failed: {e}") + import traceback + traceback.print_exc() + raise + finally: + if conn: + return_db_connection(conn) + + +if __name__ == "__main__": + run_global_locations_migration() diff --git a/truecover-backend/routes/admin_boundaries.py b/truecover-backend/routes/admin_boundaries.py index b08642ff5..7756e340c 100644 --- a/truecover-backend/routes/admin_boundaries.py +++ b/truecover-backend/routes/admin_boundaries.py @@ -23,15 +23,15 @@ ) -def find_duplicate_by_external_id(cursor, campaign_id, external_id): - """Check if location exists by external_id only (for Overture imports)""" +def find_duplicate_by_external_id(cursor, external_id): + """Check if location exists globally by external_id (for Overture imports)""" if not external_id: return None cursor.execute(""" SELECT id FROM locations - WHERE campaign_id = %s AND external_id = %s + WHERE external_id = %s LIMIT 1 - """, (campaign_id, external_id)) + """, (external_id,)) result = cursor.fetchone() return str(result[0]) if result else None @@ -393,10 +393,10 @@ def import_overture_buildings(user, pcode): from routes.locations import calculate_quadkey import mercantile - # Get indicators for this area's project (needed for coverage creation) + # Get indicators for this campaign's project (needed for coverage creation) cursor.execute(""" SELECT id FROM indicators WHERE project_id = ( - SELECT project_id FROM areas WHERE id = %s + SELECT project_id FROM campaigns WHERE id = %s ) """, (campaign_id,)) indicators = [row[0] for row in cursor.fetchall()] @@ -442,8 +442,8 @@ def import_overture_buildings(user, pcode): if lat is None or lng is None: continue - # Check for duplicate by external_id only - duplicate_id = find_duplicate_by_external_id(cursor, campaign_id, overture_id) + # Check for duplicate by external_id globally + duplicate_id = find_duplicate_by_external_id(cursor, overture_id) if duplicate_id: batch_duplicates += 1 continue @@ -467,7 +467,6 @@ def import_overture_buildings(user, pcode): # Add to batch batch_to_insert.append(( - campaign_id, overture_id, # external_id geometry_wkt, lat, @@ -495,13 +494,13 @@ def import_overture_buildings(user, pcode): batch_ids = [] result = execute_values(cursor, """ INSERT INTO locations ( - campaign_id, external_id, geometry, latitude, longitude, quadkey, properties + external_id, geometry, latitude, longitude, quadkey, properties ) VALUES %s RETURNING id - """, [(campaign_id, ext_id, f"SRID=4326;{geom}", lat, lng, qk, props) - for campaign_id, ext_id, geom, lat, lng, qk, props in batch_to_insert], - template="(%s, %s, ST_GeomFromText(%s), %s, %s, %s, %s)", + """, [(ext_id, f"SRID=4326;{geom}", lat, lng, qk, props) + for ext_id, geom, lat, lng, qk, props in batch_to_insert], + template="(%s, ST_GeomFromText(%s), %s, %s, %s, %s)", fetch=True) # Get all returned IDs @@ -541,7 +540,7 @@ def import_overture_buildings(user, pcode): cursor.executemany(""" INSERT INTO coverage (location_id, campaign_id, indicator_id, version, n_trials, n_covered, quadkey) VALUES (%s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (location_id, indicator_id, version) DO NOTHING + ON CONFLICT (campaign_id, location_id, indicator_id, version) DO NOTHING """, coverage_batch) coverage_created = cursor.rowcount @@ -587,15 +586,15 @@ def import_overture_buildings(user, pcode): pixel_wkt = f'POLYGON(({bounds.west} {bounds.south},{bounds.east} {bounds.south},{bounds.east} {bounds.north},{bounds.west} {bounds.north},{bounds.west} {bounds.south}))' pixels_to_insert.append(( - campaign_id, quadkey, pixel_wkt, center_lat, center_lng, 18 + quadkey, pixel_wkt, center_lat, center_lng, 18 )) # Batch insert with ON CONFLICT DO NOTHING to handle duplicates if pixels_to_insert: cursor.executemany(""" - INSERT INTO pixels (campaign_id, quadkey, geometry, latitude, longitude, level) - VALUES (%s, %s, ST_GeomFromText(%s, 4326), %s, %s, %s) - ON CONFLICT ON CONSTRAINT pixels_area_quadkey_unique DO NOTHING + INSERT INTO pixels (quadkey, geometry, latitude, longitude, level) + VALUES (%s, ST_GeomFromText(%s, 4326), %s, %s, %s) + ON CONFLICT (quadkey) DO NOTHING """, pixels_to_insert) # Count how many were actually inserted @@ -607,7 +606,7 @@ def import_overture_buildings(user, pcode): if pixels_created > 0: cursor.execute(""" SELECT id FROM indicators WHERE project_id = ( - SELECT project_id FROM areas WHERE id = %s + SELECT project_id FROM campaigns WHERE id = %s ) """, (campaign_id,)) @@ -619,12 +618,12 @@ def import_overture_buildings(user, pcode): INSERT INTO coverage_pixel (quadkey, indicator_id, campaign_id, version, n_trials, n_covered) SELECT p.quadkey, %s, %s, 0, 0, 0 FROM pixels p - WHERE p.campaign_id = %s + WHERE p.quadkey = ANY(%s) AND NOT EXISTS ( SELECT 1 FROM coverage_pixel cp WHERE cp.quadkey = p.quadkey AND cp.indicator_id = %s AND cp.campaign_id = %s ) - """, (indicator_id, campaign_id, campaign_id, indicator_id, campaign_id)) + """, (indicator_id, campaign_id, list(unique_quadkeys), indicator_id, campaign_id)) conn.commit() print(f"Populated coverage_pixel for {pixels_created} new pixels") diff --git a/truecover-backend/routes/campaigns.py b/truecover-backend/routes/campaigns.py index 809e9f0b8..c118f33fc 100644 --- a/truecover-backend/routes/campaigns.py +++ b/truecover-backend/routes/campaigns.py @@ -581,19 +581,17 @@ def compute_pixels_for_area(user, area_id): WITH pixel_stats AS ( SELECT COUNT(*) as pixel_count, - COALESCE(SUM(p.population), 0) as total_population + COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population FROM pixel_area pa JOIN pixels p ON pa.quadkey = p.quadkey + LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey WHERE pa.campaign_area_id = %s ), location_counts AS ( - SELECT COUNT(l.id) as building_count - FROM campaign_areas ca - LEFT JOIN locations l ON l.campaign_id = ca.campaign_id - AND l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat - AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng - AND ST_Intersects(l.geometry, ca.geometry) - WHERE ca.id = %s + SELECT COUNT(DISTINCT l.id) as building_count + FROM pixel_area pa2 + LEFT JOIN locations l ON l.quadkey = pa2.quadkey + WHERE pa2.campaign_area_id = %s ) UPDATE campaign_areas SET @@ -603,13 +601,41 @@ def compute_pixels_for_area(user, area_id): WHERE id = %s """, (area_id, area_id, area_id)) + # Create coverage_pixel records for all project indicators + cursor.execute(""" + SELECT i.id FROM indicators i + JOIN campaigns c ON i.project_id = c.project_id + WHERE c.id = %s + """, (campaign_id,)) + indicator_ids = [row[0] for row in cursor.fetchall()] + + coverage_pixel_count = 0 + for ind_id in indicator_ids: + cursor.execute(""" + INSERT INTO coverage_pixel ( + campaign_id, indicator_id, quadkey, version, + n_trials, n_covered, + exceedance_probability, exceedance_uncertainty, + prevalence_prediction, prevalence_bci_width + ) + SELECT %s, %s, pa.quadkey, 0, 0, 0, 0.5, 0.5, 0.5, 0.5 + FROM pixel_area pa + JOIN pixels p ON pa.quadkey = p.quadkey + WHERE pa.campaign_area_id = %s + ON CONFLICT (quadkey, indicator_id, campaign_id, version) DO NOTHING + """, (campaign_id, ind_id, area_id)) + coverage_pixel_count += cursor.rowcount + + print(f"Created {coverage_pixel_count} coverage_pixel records for area {area_id}") + conn.commit() cursor.close() return jsonify({ 'success': True, 'campaign_area_id': area_id, - 'pixels_computed': inserted_count + 'pixels_computed': inserted_count, + 'coverage_pixels_created': coverage_pixel_count }), 200 except Exception as e: @@ -670,19 +696,17 @@ def compute_all_pixels_for_campaign(user, campaign_id): WITH pixel_stats AS ( SELECT COUNT(*) as pixel_count, - COALESCE(SUM(p.population), 0) as total_population + COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population FROM pixel_area pa JOIN pixels p ON pa.quadkey = p.quadkey + LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey WHERE pa.campaign_area_id = %s ), location_counts AS ( - SELECT COUNT(l.id) as building_count - FROM campaign_areas ca - LEFT JOIN locations l ON l.campaign_id = ca.campaign_id - AND l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat - AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng - AND ST_Intersects(l.geometry, ca.geometry) - WHERE ca.id = %s + SELECT COUNT(DISTINCT l.id) as building_count + FROM pixel_area pa2 + LEFT JOIN locations l ON l.quadkey = pa2.quadkey + WHERE pa2.campaign_area_id = %s ) UPDATE campaign_areas SET @@ -803,6 +827,7 @@ def sample_campaign_area(user, area_id): resample = data.get('resample', False) round_id = data.get('round_id') sample_target = data.get('sample_target', 'pixels') # 'pixels' or 'buildings' + buildings_per_pixel = data.get('buildings_per_pixel', 0) if not indicator_id: return jsonify({'error': 'indicator_id is required'}), 400 @@ -839,7 +864,8 @@ async def start_workflow(): resample, round_number, None, # round_name (not needed since round exists) - sample_target + sample_target, + buildings_per_pixel ], id=workflow_id, task_queue="truecover-tasks" diff --git a/truecover-backend/routes/coverage.py b/truecover-backend/routes/coverage.py index d235cae50..1d64bd6b4 100644 --- a/truecover-backend/routes/coverage.py +++ b/truecover-backend/routes/coverage.py @@ -237,11 +237,11 @@ def predict_coverage(user): conn = get_db_connection() cursor = conn.cursor() - # Query ALL coverage records for this indicator and area + # Query ALL coverage records for this indicator and campaign cursor.execute(""" SELECT l.id, - l.campaign_id, + c.campaign_id, ST_AsGeoJSON(l.geometry) as geometry, c.n_trials, c.n_covered, @@ -250,7 +250,7 @@ def predict_coverage(user): FROM locations l JOIN coverage c ON l.id = c.location_id WHERE c.indicator_id = %s - AND l.campaign_id = %s + AND c.campaign_id = %s """, (indicator_id, campaign_id)) location_data = cursor.fetchall() @@ -1014,7 +1014,7 @@ def list_area_coverage_pixel(user, campaign_id): where_clause += """ AND EXISTS ( SELECT 1 FROM locations l - WHERE l.quadkey = cp.quadkey AND l.campaign_id = cp.campaign_id + WHERE l.quadkey = cp.quadkey ) """ @@ -1042,7 +1042,7 @@ def list_area_coverage_pixel(user, campaign_id): FROM coverage_pixel cp JOIN indicators i ON cp.indicator_id = i.id LEFT JOIN pixels p ON cp.quadkey = p.quadkey - LEFT JOIN pixel_location_counts plc ON cp.quadkey = plc.quadkey AND cp.campaign_id = plc.campaign_id + LEFT JOIN pixel_location_counts plc ON cp.quadkey = plc.quadkey {where_clause} LIMIT %s OFFSET %s """ diff --git a/truecover-backend/routes/enrichment.py b/truecover-backend/routes/enrichment.py index e88e0389c..b4b3227dc 100644 --- a/truecover-backend/routes/enrichment.py +++ b/truecover-backend/routes/enrichment.py @@ -1,6 +1,8 @@ # ABOUTME: API endpoints for pixel enrichment jobs # ABOUTME: Provides async job creation and status tracking for COG/STAC data enrichment +import uuid as uuid_mod + from flask import Blueprint, jsonify, request from auth.middleware import require_auth from auth.helpers import check_campaign_access @@ -32,16 +34,25 @@ def create_enrichment_job(user, campaign_id): conn = get_db_connection() cursor = conn.cursor() - # Verify data source exists - cursor.execute(""" - SELECT id, default_statistic FROM data_sources WHERE id = %s - """, (data_source_id,)) + # Look up data source by UUID or by name prefix + try: + uuid_mod.UUID(data_source_id) + cursor.execute(""" + SELECT id, default_statistic FROM data_sources WHERE id = %s + """, (data_source_id,)) + except ValueError: + cursor.execute(""" + SELECT id, default_statistic FROM data_sources WHERE name LIKE %s + ORDER BY created_at DESC LIMIT 1 + """, (f"{data_source_id}%",)) ds_row = cursor.fetchone() if not ds_row: cursor.close() return jsonify({'error': 'Data source not found'}), 404 + # Use the resolved UUID for FK references + resolved_ds_id = str(ds_row[0]) default_statistic = ds_row[1] statistic = data.get('statistic', default_statistic) @@ -60,7 +71,7 @@ def create_enrichment_job(user, campaign_id): # Generate workflow ID timestamp = datetime.now().strftime('%Y%m%d%H%M%S') - workflow_id = f"pixel-enrichment-{campaign_id}-{data_source_id}-{timestamp}" + workflow_id = f"pixel-enrichment-{campaign_id}-{resolved_ds_id}-{timestamp}" # Create job with workflow_id cursor.execute(""" @@ -77,7 +88,7 @@ def create_enrichment_job(user, campaign_id): pixels_processed, pixels_total, error_message, retry_count, last_attempted_at, created_at, updated_at, workflow_id - """, (campaign_id, data_source_id, statistic, 'pending', pixels_total, workflow_id)) + """, (campaign_id, resolved_ds_id, statistic, 'pending', pixels_total, workflow_id)) row = cursor.fetchone() job_id = str(row[0]) diff --git a/truecover-backend/routes/locations.py b/truecover-backend/routes/locations.py index cbd1d37ce..81f3dbd9a 100644 --- a/truecover-backend/routes/locations.py +++ b/truecover-backend/routes/locations.py @@ -47,9 +47,9 @@ def geometry_to_wkt(geometry_dict): return None -def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): +def find_duplicate(cursor, external_id, lat, lng, geometry_wkt): """ - Find duplicate location using multiple strategies: + Find duplicate location globally using multiple strategies: 1. Match by external_id 2. Match by lat/lng (within tolerance) 3. Match by geometry intersection @@ -58,9 +58,9 @@ def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): if external_id: cursor.execute(""" SELECT id FROM locations - WHERE campaign_id = %s AND external_id = %s + WHERE external_id = %s LIMIT 1 - """, (campaign_id, external_id)) + """, (external_id,)) result = cursor.fetchone() if result: return str(result[0]) @@ -69,11 +69,10 @@ def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): if lat is not None and lng is not None: cursor.execute(""" SELECT id FROM locations - WHERE campaign_id = %s - AND ABS(latitude - %s) < 0.0001 + WHERE ABS(latitude - %s) < 0.0001 AND ABS(longitude - %s) < 0.0001 LIMIT 1 - """, (campaign_id, lat, lng)) + """, (lat, lng)) result = cursor.fetchone() if result: return str(result[0]) @@ -82,10 +81,9 @@ def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): if geometry_wkt: cursor.execute(""" SELECT id FROM locations - WHERE campaign_id = %s - AND ST_Intersects(geometry, ST_GeomFromText(%s, 4326)) + WHERE ST_Intersects(geometry, ST_GeomFromText(%s, 4326)) LIMIT 1 - """, (campaign_id, geometry_wkt)) + """, (geometry_wkt,)) result = cursor.fetchone() if result: return str(result[0]) @@ -108,9 +106,9 @@ def populate_coverage_for_locations(cursor, campaign_id, new_location_ids): return try: - # Get the project_id from the area + # Get the project_id from the campaign cursor.execute(""" - SELECT project_id FROM areas WHERE id = %s + SELECT project_id FROM campaigns WHERE id = %s """, (campaign_id,)) result = cursor.fetchone() if not result: @@ -374,8 +372,8 @@ def upload_locations(user, campaign_id): # Convert geometry to WKT for PostGIS geometry_wkt = geometry_to_wkt(geometry) if geometry else None - # Check for duplicates - duplicate_id = find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt) + # Check for duplicates globally + duplicate_id = find_duplicate(cursor, external_id, lat, lng, geometry_wkt) if duplicate_id: # Update existing location - merge properties and recalculate quadkey @@ -402,14 +400,14 @@ def upload_locations(user, campaign_id): quadkey = calculate_quadkey(lat, lng) cursor.execute(""" INSERT INTO locations ( - campaign_id, external_id, geometry, latitude, longitude, quadkey, properties + external_id, geometry, latitude, longitude, quadkey, properties ) VALUES ( - %s, %s, ST_GeomFromText(%s, 4326), %s, %s, %s, %s + %s, ST_GeomFromText(%s, 4326), %s, %s, %s, %s ) RETURNING id """, ( - campaign_id, external_id, geometry_wkt, lat, lng, quadkey, + external_id, geometry_wkt, lat, lng, quadkey, json.dumps(properties) )) new_location_id = cursor.fetchone()[0] @@ -435,18 +433,20 @@ def upload_locations(user, campaign_id): if inserted_count > 0: print(f"Auto-generating pixels for uploaded locations...") try: - # Get all unique quadkeys from locations in this area + # Get all unique quadkeys from locations in this campaign's areas cursor.execute(""" - SELECT DISTINCT quadkey FROM locations - WHERE campaign_id = %s AND quadkey IS NOT NULL + SELECT DISTINCT l.quadkey FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE ca.campaign_id = %s AND l.quadkey IS NOT NULL """, (campaign_id,)) location_quadkeys = {row[0] for row in cursor.fetchall()} if location_quadkeys: - # Get existing pixels for this area + # Get existing pixels globally cursor.execute(""" - SELECT quadkey FROM pixels WHERE campaign_id = %s - """, (campaign_id,)) + SELECT quadkey FROM pixels WHERE quadkey = ANY(%s) + """, (list(location_quadkeys),)) existing_quadkeys = {row[0] for row in cursor.fetchall()} # Find quadkeys that need pixels @@ -469,7 +469,6 @@ def upload_locations(user, campaign_id): geometry_wkt = f"POLYGON(({bounds.west} {bounds.south}, {bounds.west} {bounds.north}, {bounds.east} {bounds.north}, {bounds.east} {bounds.south}, {bounds.west} {bounds.south}))" pixel_data.append(( - campaign_id, quadkey, geometry_wkt, centroid_lat, @@ -477,11 +476,11 @@ def upload_locations(user, campaign_id): tile.z )) - # Batch insert pixels (upsert to handle any race conditions) + # Batch insert pixels (global, upsert to handle any race conditions) cursor.executemany(""" - INSERT INTO pixels (campaign_id, quadkey, geometry, latitude, longitude, level) - VALUES (%s, %s, ST_GeomFromText(%s, 4326), %s, %s, %s) - ON CONFLICT ON CONSTRAINT pixels_area_quadkey_unique DO NOTHING + INSERT INTO pixels (quadkey, geometry, latitude, longitude, level) + VALUES (%s, ST_GeomFromText(%s, 4326), %s, %s, %s) + ON CONFLICT (quadkey) DO NOTHING """, pixel_data) print(f"Auto-generated {len(pixel_data)} pixels for uploaded locations") @@ -542,24 +541,28 @@ def list_locations(user, campaign_id): limit = request.args.get('limit', type=int, default=200) offset = request.args.get('offset', type=int, default=0) - # Get total count first + # Get total count first (locations in campaign's areas via quadkey/pixel_area) cursor.execute(""" - SELECT COUNT(*) - FROM locations - WHERE campaign_id = %s + SELECT COUNT(DISTINCT l.id) + FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE ca.campaign_id = %s """, (campaign_id,)) total_count = cursor.fetchone()[0] # Return lightweight list without geometry - map uses vector tiles now cursor.execute(""" - SELECT - id, external_id, - latitude, longitude, - properties, quadkey, - created_at, updated_at - FROM locations - WHERE campaign_id = %s - ORDER BY created_at DESC + SELECT DISTINCT ON (l.id) + l.id, l.external_id, + l.latitude, l.longitude, + l.properties, l.quadkey, + l.created_at, l.updated_at + FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE ca.campaign_id = %s + ORDER BY l.id, l.created_at DESC LIMIT %s OFFSET %s """, (campaign_id, limit, offset)) @@ -617,10 +620,13 @@ def update_location(user, campaign_id, location_id): conn = get_db_connection() cursor = conn.cursor() - # Verify location belongs to area + # Verify location belongs to campaign via spatial overlap cursor.execute(""" - SELECT id FROM locations - WHERE id = %s AND campaign_id = %s + SELECT l.id FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE l.id = %s AND ca.campaign_id = %s + LIMIT 1 """, (location_id, campaign_id)) if not cursor.fetchone(): @@ -633,11 +639,10 @@ def update_location(user, campaign_id, location_id): SET external_id = COALESCE(%s, external_id), updated_at = NOW() - WHERE id = %s AND campaign_id = %s + WHERE id = %s """, ( data.get('external_id'), - location_id, - campaign_id + location_id )) conn.commit() @@ -668,10 +673,13 @@ def delete_location(user, campaign_id, location_id): conn = get_db_connection() cursor = conn.cursor() - # Verify location belongs to area + # Verify location belongs to campaign via spatial overlap cursor.execute(""" - SELECT id FROM locations - WHERE id = %s AND campaign_id = %s + SELECT l.id FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE l.id = %s AND ca.campaign_id = %s + LIMIT 1 """, (location_id, campaign_id)) if not cursor.fetchone(): @@ -680,9 +688,8 @@ def delete_location(user, campaign_id, location_id): # Delete location cursor.execute(""" - DELETE FROM locations - WHERE id = %s AND campaign_id = %s - """, (location_id, campaign_id)) + DELETE FROM locations WHERE id = %s + """, (location_id,)) conn.commit() cursor.close() diff --git a/truecover-backend/routes/rounds.py b/truecover-backend/routes/rounds.py index b633eb5b9..04ee55e8c 100644 --- a/truecover-backend/routes/rounds.py +++ b/truecover-backend/routes/rounds.py @@ -176,13 +176,13 @@ async def get_status(): @rounds_bp.route('/api/campaigns//rounds', methods=['POST']) @require_auth def create_round(user, campaign_id): - """Create a new round and run adaptive sampling using Temporal workflow""" + """Create a round record and optionally start per-area sampling workflows""" from datetime import datetime from temporal.client import get_temporal_client, run_async - from temporal.workflows.round_generation import RoundGenerationWorkflow + from temporal.workflows.campaign_area_sampling import CampaignAreaSamplingWorkflow + conn = None try: - # Check if user has access to this area if not check_campaign_access(user['id'], campaign_id): return jsonify({'error': 'Access denied'}), 403 @@ -195,13 +195,7 @@ def create_round(user, campaign_id): start_date = data.get('start_date') end_date = data.get('end_date') indicator_id = data.get('indicator_id') - batch_size = data.get('batch_size', 10) - uncertainty_field = data.get('uncertainty_field', 'exceedance_uncertainty') - allow_revisit = data.get('allow_revisit', False) - sampling_target = data.get('sampling_target', 'locations') - admin_pcode = data.get('admin_pcode') - min_population = data.get('min_population') - population_field = data.get('population_field') + sample_areas = data.get('sample_areas', []) if not name: return jsonify({'error': 'Round name is required'}), 400 @@ -209,41 +203,90 @@ def create_round(user, campaign_id): if not indicator_id: return jsonify({'error': 'Indicator ID is required'}), 400 - # Generate workflow ID - timestamp = datetime.now().strftime('%Y%m%d%H%M%S') - workflow_id = f"round-generation-{campaign_id}-{timestamp}" + conn = get_db_connection() + cursor = conn.cursor() - # Start workflow - async def start_workflow(): - client = await get_temporal_client() - handle = await client.start_workflow( - RoundGenerationWorkflow.run, - args=[ - campaign_id, name, description, start_date, end_date, - indicator_id, batch_size, uncertainty_field, - allow_revisit, sampling_target, admin_pcode, - min_population, population_field - ], - id=workflow_id, - task_queue="truecover-tasks" - ) - return handle + # Get next round number + cursor.execute(""" + SELECT COALESCE(MAX(round_number), 0) + 1 + FROM rounds + WHERE campaign_id = %s + """, (campaign_id,)) + round_number = cursor.fetchone()[0] - run_async(start_workflow()) + # Create the round + cursor.execute(""" + INSERT INTO rounds (campaign_id, round_number, name, description, start_date, end_date, indicator_id, sampling_target) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id, round_number, name, description, start_date, end_date, created_at, updated_at, sampling_target + """, (campaign_id, round_number, name, description, start_date, end_date, indicator_id, 'pixels')) - print(f"Started round generation workflow: {workflow_id}") + round_data = cursor.fetchone() + round_id = str(round_data[0]) + + conn.commit() + cursor.close() + + # Start per-area sampling workflows if requested + workflow_ids = {} + if sample_areas: + timestamp = datetime.now().strftime('%Y%m%d%H%M%S') + + async def start_area_workflows(): + client = await get_temporal_client() + for area in sample_areas: + area_id = area['area_id'] + sample_count = area.get('sample_count', 50) + sample_target = area.get('sample_target', 'pixels') + buildings_per_pixel = area.get('buildings_per_pixel', 0) + wf_id = f"area-sampling-{area_id}-round{round_number}-{timestamp}" + + await client.start_workflow( + CampaignAreaSamplingWorkflow.run, + args=[ + campaign_id, + indicator_id, + area_id, + sample_count, + False, # resample + round_number, + None, # round_name + sample_target, + buildings_per_pixel, + ], + id=wf_id, + task_queue="truecover-tasks" + ) + workflow_ids[area_id] = wf_id + print(f"Started area sampling workflow: {wf_id}") + + run_async(start_area_workflows()) return jsonify({ - 'workflow_id': workflow_id, - 'status': 'started', - 'message': 'Round generation started. Use the workflow_id to check progress.' - }), 202 + 'round': { + 'id': round_id, + 'round_number': round_data[1], + 'name': round_data[2], + 'description': round_data[3], + 'start_date': round_data[4].isoformat() if round_data[4] else None, + 'end_date': round_data[5].isoformat() if round_data[5] else None, + 'created_at': round_data[6].isoformat() if round_data[6] else None, + 'updated_at': round_data[7].isoformat() if round_data[7] else None, + 'sampling_target': round_data[8], + }, + 'workflow_ids': workflow_ids, + }), 201 except Exception as e: - print(f"Error starting round generation workflow: {e}") + if conn: + conn.rollback() + print(f"Error creating round: {e}") import traceback traceback.print_exc() - return jsonify({'error': 'Failed to start round generation', 'details': str(e)}), 500 + return jsonify({'error': 'Failed to create round', 'details': str(e)}), 500 + finally: + if conn: + return_db_connection(conn) @rounds_bp.route('/api/campaigns//rounds/stratified-cluster', methods=['POST']) diff --git a/truecover-backend/routes/visits.py b/truecover-backend/routes/visits.py index 64d7a31a3..5e1f608a3 100644 --- a/truecover-backend/routes/visits.py +++ b/truecover-backend/routes/visits.py @@ -73,12 +73,11 @@ def create_visits_bulk(user): actual_location_id = None match_type = None - # Step 1: Check if uploaded_location_id matches existing location by ID + # Step 1: Check if uploaded_location_id matches existing location by ID (UUID is globally unique) if uploaded_location_id: cursor.execute(""" - SELECT id FROM locations - WHERE id = %s AND campaign_id = %s - """, (uploaded_location_id, campaign_id)) + SELECT id FROM locations WHERE id = %s + """, (uploaded_location_id,)) location_result = cursor.fetchone() if location_result: @@ -86,17 +85,19 @@ def create_visits_bulk(user): matched_by_id += 1 match_type = 'id' - # Step 2: If no ID match, try proximity match (within 50 meters) + # Step 2: If no ID match, try proximity match (within 50 meters) scoped to campaign areas if not actual_location_id: cursor.execute(""" - SELECT id, ST_Distance( - geometry::geography, + SELECT l.id, ST_Distance( + l.geometry::geography, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography ) as distance - FROM locations - WHERE campaign_id = %s + FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE ca.campaign_id = %s AND ST_DWithin( - geometry::geography, + l.geometry::geography, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, 50 ) @@ -124,10 +125,10 @@ def create_visits_bulk(user): if quadkey: affected_quadkeys.add(quadkey) cursor.execute(""" - INSERT INTO locations (campaign_id, external_id, latitude, longitude, geometry, quadkey) - VALUES (%s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s) + INSERT INTO locations (external_id, latitude, longitude, geometry, quadkey) + VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s) RETURNING id - """, (campaign_id, uploaded_location_id, latitude, longitude, longitude, latitude, quadkey)) + """, (uploaded_location_id, latitude, longitude, longitude, latitude, quadkey)) location_result = cursor.fetchone() actual_location_id = str(location_result[0]) diff --git a/truecover-backend/temporal/activities/cluster_sampling.py b/truecover-backend/temporal/activities/cluster_sampling.py index 3c28c0ca6..eefadaa0a 100644 --- a/truecover-backend/temporal/activities/cluster_sampling.py +++ b/truecover-backend/temporal/activities/cluster_sampling.py @@ -374,13 +374,10 @@ async def compute_pixels_for_campaign_areas( WHERE pa.campaign_area_id = %s ), location_counts AS ( - SELECT COUNT(l.id) as building_count - FROM campaign_areas ca - LEFT JOIN locations l ON l.campaign_id = ca.campaign_id - AND l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat - AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng - AND ST_Intersects(l.geometry, ca.geometry) - WHERE ca.id = %s + SELECT COUNT(DISTINCT l.id) as building_count + FROM pixel_area pa2 + LEFT JOIN locations l ON l.quadkey = pa2.quadkey + WHERE pa2.campaign_area_id = %s ) UPDATE campaign_areas SET cached_pixel_count = (SELECT pixel_count FROM pixel_stats), @@ -621,7 +618,22 @@ async def sample_pixels_for_campaign_area( ) response.raise_for_status() - result = response.json() + # R function may print stdout before/after JSON — extract just the JSON + response_text = response.text + json_start = response_text.find('{') + if json_start == -1: + raise ValueError(f"No JSON in adaptive sampling response: {response_text[:200]}") + depth = 0 + json_end = json_start + for i, char in enumerate(response_text[json_start:], start=json_start): + if char == '{': + depth += 1 + elif char == '}': + depth -= 1 + if depth == 0: + json_end = i + 1 + break + result = json.loads(response_text[json_start:json_end]) # Unwrap if wrapped by OpenFaaS if isinstance(result, dict) and result.get('function_status') == 'success' and result.get('result'): @@ -685,8 +697,10 @@ async def assign_pixels_to_round( COALESCE(SUM(p.population), 0) as sampled_pop FROM coverage_pixel cp JOIN pixel_area pa ON cp.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id JOIN pixels p ON cp.quadkey = p.quadkey WHERE pa.campaign_area_id = %s + AND cp.campaign_id = ca.campaign_id AND cp.rounds IS NOT NULL AND array_length(cp.rounds, 1) > 0 ) @@ -894,23 +908,20 @@ async def sample_buildings_for_campaign_area( # Fetch buildings (locations) within this campaign area that haven't been sampled cursor.execute(""" - SELECT + SELECT DISTINCT ON (l.id) l.id as location_id, l.quadkey, l.latitude, l.longitude, c.exceedance_probability, c.exceedance_uncertainty, c.prevalence_bci_width, c.prevalence_prediction FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey LEFT JOIN coverage c ON c.location_id = l.id AND c.campaign_id = %s AND c.indicator_id = %s - JOIN campaign_areas ca ON l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat - AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng - AND ST_Intersects(l.geometry, ca.geometry) - WHERE l.campaign_id = %s - AND ca.id = %s + WHERE pa.campaign_area_id = %s AND (c.rounds IS NULL OR array_length(c.rounds, 1) IS NULL OR array_length(c.rounds, 1) = 0) - """, (campaign_id, indicator_id, campaign_id, campaign_area_id)) + """, (campaign_id, indicator_id, campaign_area_id)) records = cursor.fetchall() @@ -1060,9 +1071,8 @@ async def assign_buildings_to_round( SELECT COUNT(DISTINCT c.location_id) as cnt FROM coverage c JOIN locations l ON c.location_id = l.id - JOIN campaign_areas ca ON l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat - AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng - WHERE ca.id = %s + JOIN pixel_area pa ON l.quadkey = pa.quadkey + WHERE pa.campaign_area_id = %s AND c.campaign_id = %s AND c.rounds IS NOT NULL AND array_length(c.rounds, 1) > 0 @@ -1084,6 +1094,259 @@ async def assign_buildings_to_round( return_db_connection(conn) +@activity.defn +async def sample_buildings_within_pixels( + campaign_id: str, + indicator_id: str, + campaign_area_id: str, + candidate_pixel_ids: List[str], + target_pixel_count: int, + buildings_per_pixel: int, + round_number: int +) -> Dict[str, Any]: + """ + Filter candidate pixels by minimum building count, assign qualified pixels + to the round, and sample buildings within them. + + candidate_pixel_ids are ordered by priority from adaptive sampling (highest + uncertainty/spatial diversity first). Includes backup pixels beyond + target_pixel_count. We walk in priority order and keep the first + target_pixel_count pixels that have >= buildings_per_pixel buildings. + """ + import requests + import os + + SAMPLING_URL = os.getenv('DOCKER_FN_SAMPLING_URL', 'http://localhost:8083') + + conn = None + try: + conn = get_db_connection() + cursor = conn.cursor() + + # Phase 1: Map pixel IDs to quadkeys, preserving candidate order + cursor.execute(""" + SELECT id, quadkey FROM coverage_pixel WHERE id = ANY(%s::uuid[]) + """, (candidate_pixel_ids,)) + pixel_id_to_quadkey = {str(row[0]): row[1] for row in cursor.fetchall()} + + all_quadkeys = list(set(pixel_id_to_quadkey.values())) + if not all_quadkeys: + return { + 'pixels_assigned': 0, 'pixels_skipped': 0, + 'buildings_selected': 0, 'pixels_with_buildings': 0 + } + + # Phase 2: Count buildings per quadkey (locations are global) + cursor.execute(""" + SELECT l.quadkey, COUNT(*) as cnt + FROM locations l + WHERE l.quadkey = ANY(%s) + GROUP BY l.quadkey + """, (all_quadkeys,)) + quadkey_building_count = {row[0]: row[1] for row in cursor.fetchall()} + + # Phase 3: Filter pixels — walk in priority order, keep those meeting threshold + qualified_pixel_ids = [] + qualified_quadkeys = [] + skipped = 0 + + for pixel_id in candidate_pixel_ids: + if len(qualified_pixel_ids) >= target_pixel_count: + break + qk = pixel_id_to_quadkey.get(pixel_id) + if not qk: + skipped += 1 + continue + bcount = quadkey_building_count.get(qk, 0) + if bcount >= buildings_per_pixel: + qualified_pixel_ids.append(pixel_id) + qualified_quadkeys.append(qk) + else: + skipped += 1 + activity.logger.info( + f"Skipped pixel {pixel_id} (quadkey {qk}): " + f"only {bcount} buildings, need {buildings_per_pixel}" + ) + + activity.logger.info( + f"Pixel filtering: {len(qualified_pixel_ids)} qualified, " + f"{skipped} skipped out of {len(candidate_pixel_ids)} candidates" + ) + + if not qualified_pixel_ids: + return { + 'pixels_assigned': 0, 'pixels_skipped': skipped, + 'buildings_selected': 0, 'pixels_with_buildings': 0 + } + + # Phase 4: Assign qualified pixels to round + cursor.execute(""" + UPDATE coverage_pixel + SET rounds = array_append(COALESCE(rounds, '{}'), %s), + updated_at = NOW() + WHERE id = ANY(%s::uuid[]) AND NOT (%s = ANY(COALESCE(rounds, '{}'))) + """, (round_number, qualified_pixel_ids, round_number)) + + # Update cached_sampled_count and cached_sampled_population + cursor.execute(""" + WITH sampled AS ( + SELECT + COUNT(DISTINCT cp.quadkey) as cnt, + COALESCE(SUM(p.population), 0) as sampled_pop + FROM coverage_pixel cp + JOIN pixel_area pa ON cp.quadkey = pa.quadkey + JOIN pixels p ON cp.quadkey = p.quadkey + WHERE pa.campaign_area_id = %s + AND cp.campaign_id = %s + AND cp.rounds IS NOT NULL + AND array_length(cp.rounds, 1) > 0 + ) + UPDATE campaign_areas + SET cached_sampled_count = (SELECT cnt FROM sampled), + cached_sampled_population = (SELECT sampled_pop FROM sampled), + updated_at = NOW() + WHERE id = %s + """, (campaign_area_id, campaign_id, campaign_area_id)) + + # Phase 5: Sample buildings within qualified pixels + unique_quadkeys = list(set(qualified_quadkeys)) + cursor.execute(""" + SELECT l.id, l.quadkey, l.latitude, l.longitude, + COALESCE(c.prevalence_bci_width, 0.5) as uncertainty + FROM locations l + LEFT JOIN coverage c ON c.location_id = l.id + AND c.campaign_id = %s AND c.indicator_id = %s + WHERE l.quadkey = ANY(%s) + """, (campaign_id, indicator_id, unique_quadkeys)) + + rows = cursor.fetchall() + by_quadkey: Dict[str, list] = {} + for row in rows: + qk = row[1] + if qk not in by_quadkey: + by_quadkey[qk] = [] + by_quadkey[qk].append({ + 'id': str(row[0]), + 'lat': float(row[2]), + 'lon': float(row[3]), + 'uncertainty': float(row[4]) + }) + + total_selected = 0 + pixels_with_buildings = 0 + + for qk, buildings in by_quadkey.items(): + if not buildings: + continue + + pixels_with_buildings += 1 + + if len(buildings) <= buildings_per_pixel: + selected_ids = [b['id'] for b in buildings] + else: + coordinates = [[b['lon'], b['lat']] for b in buildings] + uncertainty_values = [b['uncertainty'] for b in buildings] + + payload = { + 'coordinates': coordinates, + 'uncertainty': uncertainty_values, + 'batch_size': buildings_per_pixel + } + + try: + response = requests.post( + SAMPLING_URL, + json=payload, + headers={'Content-Type': 'application/json'}, + timeout=60 + ) + response.raise_for_status() + + response_text = response.text + json_start = response_text.find('{') + if json_start == -1: + activity.logger.warning(f"No JSON in sampling response for quadkey {qk}") + continue + + depth = 0 + json_end = json_start + for i, char in enumerate(response_text[json_start:], start=json_start): + if char == '{': + depth += 1 + elif char == '}': + depth -= 1 + if depth == 0: + json_end = i + 1 + break + + result = json.loads(response_text[json_start:json_end]) + + if isinstance(result, dict) and result.get('function_status') == 'success' and result.get('result'): + result = result['result'] + + selected_indices = result.get('selected_indices', []) + selected_ids = [buildings[i]['id'] for i in selected_indices] + except Exception as e: + activity.logger.warning(f"Sampling failed for quadkey {qk}: {e}, selecting first {buildings_per_pixel}") + selected_ids = [b['id'] for b in buildings[:buildings_per_pixel]] + + for location_id in selected_ids: + cursor.execute(""" + INSERT INTO coverage ( + campaign_id, indicator_id, location_id, version, + n_trials, n_covered, rounds, + exceedance_probability, exceedance_uncertainty, + prevalence_prediction, prevalence_bci_width + ) + VALUES (%s, %s, %s, 0, 0, 0, ARRAY[%s], 0.5, 0.5, 0.5, 0.5) + ON CONFLICT (campaign_id, location_id, indicator_id, version) + DO UPDATE SET + rounds = array_append(COALESCE(coverage.rounds, '{}'), %s), + updated_at = NOW() + WHERE NOT (%s = ANY(COALESCE(coverage.rounds, '{}'))) + """, (campaign_id, indicator_id, location_id, round_number, round_number, round_number)) + + total_selected += len(selected_ids) + + # Update cached_sampled_count on campaign_area (buildings) + cursor.execute(""" + WITH sampled AS ( + SELECT COUNT(DISTINCT c.location_id) as cnt + FROM coverage c + JOIN locations l ON c.location_id = l.id + JOIN pixel_area pa ON l.quadkey = pa.quadkey + WHERE pa.campaign_area_id = %s + AND c.campaign_id = %s + AND c.rounds IS NOT NULL + AND array_length(c.rounds, 1) > 0 + ) + UPDATE campaign_areas + SET cached_sampled_count = (SELECT cnt FROM sampled), + updated_at = NOW() + WHERE id = %s + """, (campaign_area_id, campaign_id, campaign_area_id)) + + conn.commit() + + activity.logger.info( + f"Selected {total_selected} buildings across {pixels_with_buildings} pixels " + f"({len(qualified_pixel_ids)} assigned, {skipped} skipped) " + f"for campaign_area {campaign_area_id}" + ) + + return { + 'pixels_assigned': len(qualified_pixel_ids), + 'pixels_skipped': skipped, + 'buildings_selected': total_selected, + 'pixels_with_buildings': pixels_with_buildings + } + + finally: + if conn: + cursor.close() + return_db_connection(conn) + + @activity.defn async def clear_round_from_buildings( campaign_id: str, @@ -1114,10 +1377,9 @@ async def clear_round_from_buildings( SET rounds = array_remove(rounds, %s), updated_at = NOW() FROM locations l - JOIN campaign_areas ca ON l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat - AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng + JOIN pixel_area pa ON l.quadkey = pa.quadkey WHERE c.location_id = l.id - AND ca.id = %s + AND pa.campaign_area_id = %s AND c.campaign_id = %s AND c.indicator_id = %s AND %s = ANY(COALESCE(c.rounds, '{}')) @@ -1131,9 +1393,8 @@ async def clear_round_from_buildings( SELECT COUNT(DISTINCT c.location_id) as cnt FROM coverage c JOIN locations l ON c.location_id = l.id - JOIN campaign_areas ca ON l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat - AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng - WHERE ca.id = %s + JOIN pixel_area pa ON l.quadkey = pa.quadkey + WHERE pa.campaign_area_id = %s AND c.campaign_id = %s AND c.rounds IS NOT NULL AND array_length(c.rounds, 1) > 0 @@ -1204,6 +1465,7 @@ async def clear_round_from_pixels( JOIN pixel_area pa ON cp.quadkey = pa.quadkey JOIN pixels p ON cp.quadkey = p.quadkey WHERE pa.campaign_area_id = %s + AND cp.campaign_id = %s AND cp.rounds IS NOT NULL AND array_length(cp.rounds, 1) > 0 ) @@ -1212,7 +1474,7 @@ async def clear_round_from_pixels( cached_sampled_population = (SELECT sampled_pop FROM sampled), updated_at = NOW() WHERE id = %s - """, (campaign_area_id, campaign_area_id)) + """, (campaign_area_id, campaign_id, campaign_area_id)) conn.commit() diff --git a/truecover-backend/temporal/activities/coverage.py b/truecover-backend/temporal/activities/coverage.py index 79633a16c..321dd5650 100644 --- a/truecover-backend/temporal/activities/coverage.py +++ b/truecover-backend/temporal/activities/coverage.py @@ -72,7 +72,7 @@ async def fetch_location_coverage(campaign_id: str, indicator_id: str) -> List[D SELECT c.id, l.id as location_id, - l.campaign_id, + c.campaign_id, ST_AsGeoJSON(l.geometry) as geometry, c.n_trials, c.n_covered, @@ -81,7 +81,7 @@ async def fetch_location_coverage(campaign_id: str, indicator_id: str) -> List[D FROM coverage c JOIN locations l ON c.location_id = l.id WHERE c.indicator_id = %s - AND l.campaign_id = %s + AND c.campaign_id = %s """, (indicator_id, campaign_id)) records = cursor.fetchall() diff --git a/truecover-backend/temporal/activities/enrichment.py b/truecover-backend/temporal/activities/enrichment.py index 9a58a2653..f14bd3cac 100644 --- a/truecover-backend/temporal/activities/enrichment.py +++ b/truecover-backend/temporal/activities/enrichment.py @@ -271,16 +271,22 @@ async def enrich_area_pixels( )) conn.commit() - # Get total pixel count + # Count pixels that still need enrichment (skip already-enriched) cursor.execute(""" - SELECT COUNT(*) FROM pixels WHERE campaign_id = %s - """, (campaign_id,)) + SELECT COUNT(DISTINCT p.quadkey) + FROM pixels p + JOIN pixel_area pa ON p.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey + WHERE ca.campaign_id = %s + AND (pm.metadata IS NULL OR NOT pm.metadata ? %s) + """, (campaign_id, metadata_field_name)) total_pixels = cursor.fetchone()[0] - activity.logger.info(f"Processing {total_pixels} pixels for area {campaign_id}") + activity.logger.info(f"Processing {total_pixels} unenriched pixels for campaign {campaign_id}") if total_pixels == 0: - return {"pixels_total": 0, "pixels_updated": 0} + return {"pixels_total": 0, "pixels_updated": 0, "skipped": True} # Process in batches using cursor-based pagination batch_size = 500 @@ -288,14 +294,18 @@ async def enrich_area_pixels( offset = 0 while offset < total_pixels: - # Fetch batch of pixels + # Fetch batch of pixels that don't yet have this metadata field cursor.execute(""" - SELECT quadkey, ST_AsText(geometry) as wkt_geometry - FROM pixels - WHERE campaign_id = %s - ORDER BY quadkey + SELECT DISTINCT p.quadkey, ST_AsText(p.geometry) as wkt_geometry + FROM pixels p + JOIN pixel_area pa ON p.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey + WHERE ca.campaign_id = %s + AND (pm.metadata IS NULL OR NOT pm.metadata ? %s) + ORDER BY p.quadkey LIMIT %s OFFSET %s - """, (campaign_id, batch_size, offset)) + """, (campaign_id, metadata_field_name, batch_size, offset)) batch_rows = cursor.fetchall() if not batch_rows: @@ -337,6 +347,25 @@ async def enrich_area_pixels( activity.logger.info(f"Batch {batch_num}: extracted stats for {len(updates)}/{len(batch_rows)} pixels") offset += batch_size + # Update cached_population on affected campaign areas + if metadata_field_name == 'population' and total_updated > 0: + cursor.execute(""" + UPDATE campaign_areas ca + SET cached_population = sub.total_pop + FROM ( + SELECT pa.campaign_area_id, + COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_pop + FROM pixel_area pa + JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey + JOIN campaign_areas ca2 ON pa.campaign_area_id = ca2.id + WHERE ca2.campaign_id = %s AND pm.metadata ? 'population' + GROUP BY pa.campaign_area_id + ) sub + WHERE ca.id = sub.campaign_area_id + """, (campaign_id,)) + conn.commit() + activity.logger.info(f"Updated cached_population for campaign areas") + activity.logger.info(f"Enrichment complete: {total_updated}/{total_pixels} pixels updated") return {"pixels_total": total_pixels, "pixels_updated": total_updated} diff --git a/truecover-backend/temporal/activities/locations.py b/truecover-backend/temporal/activities/locations.py index bcdefc1cb..e72d83110 100644 --- a/truecover-backend/temporal/activities/locations.py +++ b/truecover-backend/temporal/activities/locations.py @@ -47,9 +47,9 @@ def geometry_to_wkt(geometry_dict): return None -def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): +def find_duplicate(cursor, external_id, lat, lng, geometry_wkt): """ - Find duplicate location using multiple strategies: + Find duplicate location globally using multiple strategies: 1. Match by external_id 2. Match by lat/lng (within tolerance) 3. Match by geometry intersection @@ -58,9 +58,9 @@ def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): if external_id: cursor.execute(""" SELECT id FROM locations - WHERE campaign_id = %s AND external_id = %s + WHERE external_id = %s LIMIT 1 - """, (campaign_id, external_id)) + """, (external_id,)) result = cursor.fetchone() if result: return str(result[0]) @@ -69,11 +69,10 @@ def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): if lat is not None and lng is not None: cursor.execute(""" SELECT id FROM locations - WHERE campaign_id = %s - AND ABS(latitude - %s) < 0.0001 + WHERE ABS(latitude - %s) < 0.0001 AND ABS(longitude - %s) < 0.0001 LIMIT 1 - """, (campaign_id, lat, lng)) + """, (lat, lng)) result = cursor.fetchone() if result: return str(result[0]) @@ -82,10 +81,9 @@ def find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt): if geometry_wkt: cursor.execute(""" SELECT id FROM locations - WHERE campaign_id = %s - AND ST_Intersects(geometry, ST_GeomFromText(%s, 4326)) + WHERE ST_Intersects(geometry, ST_GeomFromText(%s, 4326)) LIMIT 1 - """, (campaign_id, geometry_wkt)) + """, (geometry_wkt,)) result = cursor.fetchone() if result: return str(result[0]) @@ -231,8 +229,8 @@ async def process_location_batch( # Convert geometry to WKT for PostGIS geometry_wkt = geometry_to_wkt(geometry) if geometry else None - # Check for duplicates - duplicate_id = find_duplicate(cursor, campaign_id, external_id, lat, lng, geometry_wkt) + # Check for duplicates globally + duplicate_id = find_duplicate(cursor, external_id, lat, lng, geometry_wkt) if duplicate_id: # Update existing location @@ -259,14 +257,14 @@ async def process_location_batch( quadkey = calculate_quadkey(lat, lng) cursor.execute(""" INSERT INTO locations ( - campaign_id, external_id, geometry, latitude, longitude, quadkey, properties + external_id, geometry, latitude, longitude, quadkey, properties ) VALUES ( - %s, %s, ST_GeomFromText(%s, 4326), %s, %s, %s, %s + %s, ST_GeomFromText(%s, 4326), %s, %s, %s, %s ) RETURNING id """, ( - campaign_id, external_id, geometry_wkt, lat, lng, quadkey, + external_id, geometry_wkt, lat, lng, quadkey, json.dumps(properties) )) new_location_id = cursor.fetchone()[0] @@ -366,10 +364,10 @@ async def populate_coverage_for_locations(campaign_id: str, new_location_ids: Li prevalence_bci_width, prevalence_prediction ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (location_id, indicator_id) DO NOTHING + ON CONFLICT (campaign_id, location_id, indicator_id, version) DO NOTHING """, ( location_id, campaign_id, indicator_id, quadkey, - 1, 0, 0, # version=1, n_trials=0, n_covered=0 for new records + 0, 0, 0, # version=0, n_trials=0, n_covered=0 for new records exceedance_probability, exceedance_uncertainty, prevalence_bci_width, prevalence_prediction )) @@ -401,10 +399,12 @@ async def generate_pixels_for_quadkeys(campaign_id: str) -> Dict[str, Any]: new_quadkeys = [] try: - # Get all unique quadkeys from locations in this area + # Get all unique quadkeys from locations in this campaign's areas cursor.execute(""" - SELECT DISTINCT quadkey FROM locations - WHERE campaign_id = %s AND quadkey IS NOT NULL + SELECT DISTINCT l.quadkey FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE ca.campaign_id = %s AND l.quadkey IS NOT NULL """, (campaign_id,)) location_quadkeys = {row[0] for row in cursor.fetchall()} diff --git a/truecover-backend/temporal/activities/overture.py b/truecover-backend/temporal/activities/overture.py index 941e13068..c0d3122e3 100644 --- a/truecover-backend/temporal/activities/overture.py +++ b/truecover-backend/temporal/activities/overture.py @@ -154,6 +154,7 @@ async def fetch_and_insert_overture_buildings( total_inserted = 0 total_duplicates = 0 all_new_location_ids = [] + all_existing_location_ids = [] try: # Query to get all buildings in one pass @@ -195,12 +196,14 @@ async def fetch_and_insert_overture_buildings( if centroid_lat is None or centroid_lng is None: continue - # Check for duplicate by external_id + # Check for duplicate by external_id (global — any campaign) pg_cursor.execute( - "SELECT id FROM locations WHERE campaign_id = %s AND external_id = %s", - (campaign_id, overture_id) + "SELECT id FROM locations WHERE external_id = %s", + (overture_id,) ) - if pg_cursor.fetchone(): + existing = pg_cursor.fetchone() + if existing: + all_existing_location_ids.append(str(existing[0])) batch_duplicates += 1 continue @@ -219,7 +222,6 @@ async def fetch_and_insert_overture_buildings( properties = {k: v for k, v in properties.items() if v is not None} batch_to_insert.append(( - campaign_id, overture_id, f"SRID=4326;{geometry_wkt}", centroid_lat, @@ -234,12 +236,12 @@ async def fetch_and_insert_overture_buildings( if batch_to_insert: result = execute_values(pg_cursor, """ INSERT INTO locations ( - campaign_id, external_id, geometry, latitude, longitude, quadkey, properties + external_id, geometry, latitude, longitude, quadkey, properties ) VALUES %s RETURNING id """, batch_to_insert, - template="(%s, %s, ST_GeomFromText(%s), %s, %s, %s, %s)", + template="(%s, ST_GeomFromText(%s), %s, %s, %s, %s)", fetch=True) new_ids = [str(row[0]) for row in result] @@ -262,7 +264,8 @@ async def fetch_and_insert_overture_buildings( 'total_fetched': total_fetched, 'inserted': total_inserted, 'duplicates': total_duplicates, - 'new_location_ids': all_new_location_ids + 'new_location_ids': all_new_location_ids, + 'existing_location_ids': all_existing_location_ids } except Exception as e: @@ -295,11 +298,10 @@ async def update_campaign_area_building_counts(campaign_id: str) -> Dict[str, in FROM ( SELECT ca2.id as area_id, - COUNT(l.id) as building_count + COUNT(DISTINCT l.id) as building_count FROM campaign_areas ca2 - LEFT JOIN admin_boundaries ab ON ab.id = ca2.admin_boundary_id - LEFT JOIN locations l ON l.campaign_id = ca2.campaign_id - AND ST_Within(l.geometry, ab.geometry) + LEFT JOIN pixel_area pa ON pa.campaign_area_id = ca2.id + LEFT JOIN locations l ON l.quadkey = pa.quadkey WHERE ca2.campaign_id = %s GROUP BY ca2.id ) counts diff --git a/truecover-backend/temporal/activities/visits.py b/truecover-backend/temporal/activities/visits.py index f0adff325..a485a6fce 100644 --- a/truecover-backend/temporal/activities/visits.py +++ b/truecover-backend/temporal/activities/visits.py @@ -60,29 +60,30 @@ async def process_visit_batch( actual_location_id = None - # Try to match by ID first + # Try to match by ID first (UUID is globally unique) if uploaded_location_id: cursor.execute(""" - SELECT id FROM locations - WHERE id = %s AND campaign_id = %s - """, (uploaded_location_id, campaign_id)) + SELECT id FROM locations WHERE id = %s + """, (uploaded_location_id,)) location_result = cursor.fetchone() if location_result: actual_location_id = str(location_result[0]) matched_by_id += 1 - # Try proximity match (within 50 meters) + # Try proximity match (within 50 meters) scoped to campaign areas if not actual_location_id: cursor.execute(""" - SELECT id, ST_Distance( - geometry::geography, + SELECT l.id, ST_Distance( + l.geometry::geography, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography ) as distance - FROM locations - WHERE campaign_id = %s + FROM locations l + JOIN pixel_area pa ON l.quadkey = pa.quadkey + JOIN campaign_areas ca ON pa.campaign_area_id = ca.id + WHERE ca.campaign_id = %s AND ST_DWithin( - geometry::geography, + l.geometry::geography, ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography, 50 ) @@ -101,10 +102,10 @@ async def process_visit_batch( if quadkey: affected_quadkeys.add(quadkey) cursor.execute(""" - INSERT INTO locations (campaign_id, external_id, latitude, longitude, geometry, quadkey) - VALUES (%s, %s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s) + INSERT INTO locations (external_id, latitude, longitude, geometry, quadkey) + VALUES (%s, %s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s) RETURNING id - """, (campaign_id, uploaded_location_id, latitude, longitude, longitude, latitude, quadkey)) + """, (uploaded_location_id, latitude, longitude, longitude, latitude, quadkey)) location_result = cursor.fetchone() actual_location_id = str(location_result[0]) diff --git a/truecover-backend/temporal/workflows/campaign_area_sampling.py b/truecover-backend/temporal/workflows/campaign_area_sampling.py index 92e3dd32d..844168279 100644 --- a/truecover-backend/temporal/workflows/campaign_area_sampling.py +++ b/truecover-backend/temporal/workflows/campaign_area_sampling.py @@ -6,6 +6,8 @@ from temporalio.common import RetryPolicy from typing import Dict, Any, Optional +BACKUP_PIXEL_COUNT = 30 + with workflow.unsafe.imports_passed_through(): from ..activities.cluster_sampling import ( create_coverage_pixels_for_campaign_area, @@ -15,6 +17,7 @@ sample_buildings_for_campaign_area, assign_buildings_to_round, clear_round_from_buildings, + sample_buildings_within_pixels, ) from ..activities.rounds import create_round_record @@ -55,12 +58,15 @@ async def run( resample: bool = False, round_number: Optional[int] = None, round_name: Optional[str] = None, - sample_target: str = 'pixels' # 'pixels' or 'buildings' + sample_target: str = 'pixels', # 'pixels' or 'buildings' + buildings_per_pixel: int = 0 ) -> Dict[str, Any]: """Run sampling for a campaign area.""" workflow.logger.info(f"Starting sampling for campaign_area {campaign_area_id}, count={sample_count}, resample={resample}, target={sample_target}") + building_result = None + retry_policy = RetryPolicy( initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=30), @@ -155,16 +161,17 @@ async def run( workflow.logger.info(f"Created {created} coverage_pixel records") # Step 4: Run adaptive sampling + # Request extra backup pixels when filtering by building count self.status = "sampling" + request_count = sample_count + BACKUP_PIXEL_COUNT if buildings_per_pixel > 0 else sample_count sampling_result = await workflow.execute_activity( sample_pixels_for_campaign_area, - args=[campaign_id, indicator_id, campaign_area_id, sample_count], + args=[campaign_id, indicator_id, campaign_area_id, request_count], start_to_close_timeout=timedelta(minutes=5), retry_policy=retry_policy ) selected_ids = sampling_result.get('selected_ids', []) - self.pixels_sampled = len(selected_ids) if not selected_ids: workflow.logger.warning(f"No pixels selected for campaign_area {campaign_area_id}") @@ -178,19 +185,37 @@ async def run( 'message': sampling_result.get('message', 'No pixels available for sampling') } - # Step 5: Assign pixels to round - self.status = "assigning_to_round" - await workflow.execute_activity( - assign_pixels_to_round, - args=[campaign_area_id, selected_ids, self.round_number], - start_to_close_timeout=timedelta(minutes=2), - retry_policy=retry_policy - ) + if buildings_per_pixel > 0: + # Combined: filter by building count, assign qualified pixels, sample buildings + self.status = "filtering_and_sampling_buildings" + building_result = await workflow.execute_activity( + sample_buildings_within_pixels, + args=[campaign_id, indicator_id, campaign_area_id, + selected_ids, sample_count, buildings_per_pixel, self.round_number], + start_to_close_timeout=timedelta(minutes=10), + retry_policy=retry_policy + ) + self.pixels_sampled = building_result.get('pixels_assigned', 0) + workflow.logger.info( + f"Building sampling: {building_result.get('buildings_selected', 0)} buildings " + f"across {building_result.get('pixels_with_buildings', 0)} pixels " + f"({building_result.get('pixels_skipped', 0)} pixels skipped)" + ) + else: + # No building threshold — assign all pixels directly + self.status = "assigning_to_round" + self.pixels_sampled = len(selected_ids) + await workflow.execute_activity( + assign_pixels_to_round, + args=[campaign_area_id, selected_ids, self.round_number], + start_to_close_timeout=timedelta(minutes=2), + retry_policy=retry_policy + ) self.status = "completed" workflow.logger.info(f"Completed sampling for campaign_area {campaign_area_id}: {self.pixels_sampled} pixels in round {self.round_number}") - return { + result = { 'campaign_area_id': campaign_area_id, 'round_number': self.round_number, 'round_id': round_id, @@ -198,3 +223,7 @@ async def run( 'total_available': sampling_result.get('total_items', 0), 'status': 'completed' } + if building_result: + result['buildings_selected'] = building_result.get('buildings_selected', 0) + result['pixels_skipped'] = building_result.get('pixels_skipped', 0) + return result diff --git a/truecover-backend/temporal/workflows/overture_import.py b/truecover-backend/temporal/workflows/overture_import.py index ae1d0d9e1..a8a0d60a4 100644 --- a/truecover-backend/temporal/workflows/overture_import.py +++ b/truecover-backend/temporal/workflows/overture_import.py @@ -101,35 +101,36 @@ async def run( self.total_inserted = import_result['inserted'] self.total_duplicates = import_result['duplicates'] all_new_location_ids = import_result['new_location_ids'] + all_existing_location_ids = import_result.get('existing_location_ids', []) workflow.logger.info( f"Import complete: fetched={self.total_fetched}, " f"inserted={self.total_inserted}, duplicates={self.total_duplicates}" ) - # Activity 3: Populate coverage for new locations - if all_new_location_ids: + # Activity 3: Populate coverage for all locations in this campaign + # Both new and existing (duplicate) buildings need coverage entries + all_location_ids = all_new_location_ids + all_existing_location_ids + if all_location_ids: self.status = 'populating_coverage' - workflow.logger.info(f"Populating coverage for {len(all_new_location_ids)} new locations") + workflow.logger.info(f"Populating coverage for {len(all_location_ids)} locations ({len(all_new_location_ids)} new, {len(all_existing_location_ids)} existing)") await workflow.execute_activity( populate_coverage_for_locations, - args=[campaign_id, all_new_location_ids], + args=[campaign_id, all_location_ids], start_to_close_timeout=timedelta(minutes=5), retry_policy=RetryPolicy(maximum_attempts=3) ) - # Activity 4: Auto-generate pixels for new quadkeys - new_quadkeys = [] - if self.total_inserted > 0: - self.status = 'generating_pixels' - workflow.logger.info("Auto-generating pixels for imported buildings") - pixel_result = await workflow.execute_activity( - generate_pixels_for_quadkeys, - args=[campaign_id], - start_to_close_timeout=timedelta(minutes=5), - retry_policy=RetryPolicy(maximum_attempts=3) - ) - new_quadkeys = pixel_result['new_quadkeys'] + # Activity 4: Auto-generate pixels for quadkeys + self.status = 'generating_pixels' + workflow.logger.info("Auto-generating pixels for imported buildings") + pixel_result = await workflow.execute_activity( + generate_pixels_for_quadkeys, + args=[campaign_id], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy(maximum_attempts=3) + ) + new_quadkeys = pixel_result['new_quadkeys'] # Activity 5: Create coverage_pixel records for new pixels if new_quadkeys: @@ -143,15 +144,14 @@ async def run( ) # Activity 6: Update campaign area building counts - if self.total_inserted > 0: - self.status = 'updating_building_counts' - workflow.logger.info("Updating campaign area building counts") - await workflow.execute_activity( - update_campaign_area_building_counts, - args=[campaign_id], - start_to_close_timeout=timedelta(minutes=2), - retry_policy=RetryPolicy(maximum_attempts=3) - ) + self.status = 'updating_building_counts' + workflow.logger.info("Updating campaign area building counts") + await workflow.execute_activity( + update_campaign_area_building_counts, + args=[campaign_id], + start_to_close_timeout=timedelta(minutes=2), + retry_policy=RetryPolicy(maximum_attempts=3) + ) self.status = 'completed' return {