From 357b4598212103fcb2d6336fcce7e51d71ca81a7 Mon Sep 17 00:00:00 2001 From: James Holcombe Date: Sun, 3 Aug 2025 22:23:21 +0100 Subject: [PATCH 1/2] Revert "lint" This reverts commit f9f38d4a5dbf7ff23bc2ce3327191d4b68c0252f. --- .../imports/detailed-changes-glide.tsx | 1 + packages/app/src/db/crud/data/index.ts | 10 +- packages/app/src/lib/imports/summary/data.ts | 187 +++++++++++++++++- 3 files changed, 191 insertions(+), 7 deletions(-) diff --git a/packages/app/src/components/imports/detailed-changes-glide.tsx b/packages/app/src/components/imports/detailed-changes-glide.tsx index cbfc137c..d47bd59a 100644 --- a/packages/app/src/components/imports/detailed-changes-glide.tsx +++ b/packages/app/src/components/imports/detailed-changes-glide.tsx @@ -14,6 +14,7 @@ import { Item, } from "@glideapps/glide-data-grid"; import { formatCellFromRawData } from "../data/data-table-panel/cell-formatter"; +import { RawCellData } from "@/hooks/use-virtualized-query-data"; type Props = { projectId: string; diff --git a/packages/app/src/db/crud/data/index.ts b/packages/app/src/db/crud/data/index.ts index 334e6a6f..b330ddde 100644 --- a/packages/app/src/db/crud/data/index.ts +++ b/packages/app/src/db/crud/data/index.ts @@ -369,7 +369,7 @@ export async function readDataRowsByUniqueKeys( } // Process each batch item's unique keys - const batchFilters = batchUniqueKeys.map((uniqueKeys) => { + const batchFilters = batchUniqueKeys.map((uniqueKeys, index) => { const itemFilters: ReturnType[] = []; // First handle parent table keys to ensure proper hierarchy @@ -401,14 +401,18 @@ export async function readDataRowsByUniqueKeys( return and(...itemFilters); }); + const queryStart = performance.now(); + // Execute query with filters const result = await query .where(and(eq(projectIdColumn, projectId), or(...batchFilters))) .execute(); + const queryTime = performance.now() - queryStart; + // Map results back to the input order with abbreviation lookups const resultMap = new Map( - result.map((row) => { + result.map((row, index) => { const processedRow = { ...row }; // Replace abbreviation IDs with their codes for display @@ -441,7 +445,7 @@ export async function readDataRowsByUniqueKeys( }), ); - const finalResults = batchUniqueKeys.map((uniqueKeys) => { + const finalResults = batchUniqueKeys.map((uniqueKeys, index) => { // Convert abbreviation IDs to codes for consistent key string generation const keyGenerationKeys = convertAbbrIdsToCodesForKeyGeneration( uniqueKeys, diff --git a/packages/app/src/lib/imports/summary/data.ts b/packages/app/src/lib/imports/summary/data.ts index a98515af..96568a15 100644 --- a/packages/app/src/lib/imports/summary/data.ts +++ b/packages/app/src/lib/imports/summary/data.ts @@ -1,3 +1,5 @@ +//@ts-nocheck + import { abbreviation } from "@common/db/schema/abbreviation"; import { GroupConfig } from "@common/db/schema/common"; import { @@ -208,7 +210,7 @@ async function preloadAbbreviations( if (!abbrMap.has(result.column)) { abbrMap.set(result.column, new Map()); } - abbrMap.get(result.column)!.set(result.code, Number(result.id)); + abbrMap.get(result.column)!.set(result.code, result.id); } } }, @@ -426,6 +428,180 @@ type KeysMap = { [key: string]: unknown; }; +// Helper function to compare records and determine if data has actually changed +function areRecordsEqual( + existing: T, + incoming: T, + excludeFields: string[] = ["id", "createdAt", "updatedAt"], + tableName?: string, + abbrMap?: Map>, +): boolean { + // Add parent ID fields to exclude list + const extendedExcludeFields = [ + ...excludeFields, + // Common parent ID patterns + ...Object.keys(existing as Record).filter( + (key) => key.endsWith("Id") && key !== "id", // Exclude parent relationship IDs but keep the main 'id' + ), + ]; + + const existingFiltered = Object.fromEntries( + Object.entries(existing as Record).filter( + ([key]) => !extendedExcludeFields.includes(key), + ), + ); + + const incomingFiltered = Object.fromEntries( + Object.entries(incoming as Record).filter( + ([key]) => !extendedExcludeFields.includes(key), + ), + ); + + // Convert abbreviation IDs to codes in existing record for fair comparison + const convertAbbrIdsToCodesInRecord = (record: Record) => { + const converted = { ...record }; + + if (abbrMap) { + for (const [column, codeMap] of abbrMap.entries()) { + const value = record[column]; + if (value !== null && value !== undefined) { + // Handle both string and number IDs + const valueStr = value.toString(); + // The abbrMap has structure: column -> Map(code -> id) + // We need to find the code for the given ID + for (const [code, id] of codeMap.entries()) { + if (id.toString() === valueStr) { + converted[column] = code; + break; + } + } + } + } + } + + return converted; + }; + + // Convert abbreviation IDs to codes in existing record + const existingWithCodes = convertAbbrIdsToCodesInRecord(existingFiltered); + + // Normalize null/undefined values for comparison + const normalizeValue = (value: unknown) => { + if (value === null || value === undefined || value === "") { + return null; + } + // Convert Date objects to ISO strings for consistent comparison + if (value instanceof Date) { + return value.toISOString(); + } + return value; + }; + + // Ensure both objects have the same set of fields for fair comparison + const getAllFieldsFromBothRecords = ( + obj1: Record, + obj2: Record, + ) => { + const allKeys = new Set([...Object.keys(obj1), ...Object.keys(obj2)]); + + const normalizeRecord = (obj: Record) => { + const normalized: Record = {}; + for (const key of allKeys) { + normalized[key] = normalizeValue(obj[key]); + } + return normalized; + }; + + return { + normalized1: normalizeRecord(obj1), + normalized2: normalizeRecord(obj2), + }; + }; + + const { normalized1: normalizedExisting, normalized2: normalizedIncoming } = + getAllFieldsFromBothRecords(existingWithCodes, incomingFiltered); + + // Use deep equality comparison for all values + const deepEqual = ( + obj1: Record, + obj2: Record, + ): boolean => { + const keys1 = Object.keys(obj1); + const keys2 = Object.keys(obj2); + + if (keys1.length !== keys2.length) { + return false; + } + + for (const key of keys1) { + if (!keys2.includes(key)) { + return false; + } + + const val1 = obj1[key]; + const val2 = obj2[key]; + + // Regular equality for all values + if (val1 !== val2) { + return false; + } + } + + return true; + }; + + const isEqual = deepEqual(normalizedExisting, normalizedIncoming); + + // DEBUG: Add targeted debugging for remaining location_details mismatches + if (!isEqual && tableName === "location_details") { + // Log all mismatches for location_details to identify the remaining 2 + console.log(`\nšŸ” DEBUG: location_details comparison mismatch:`); + console.log("Excluded fields:", extendedExcludeFields); + + // Find specific field differences + const allKeys = new Set([ + ...Object.keys(normalizedExisting), + ...Object.keys(normalizedIncoming), + ]); + + console.log("Field differences:"); + let diffCount = 0; + for (const key of allKeys) { + const existingVal = normalizedExisting[key]; + const incomingVal = normalizedIncoming[key]; + + // Special handling for numeric comparison debugging + if (typeof existingVal === "number" && typeof incomingVal === "number") { + const roundToSignificantFigures = (num: number, sf: number): number => { + if (num === 0) return 0; + const magnitude = Math.floor(Math.log10(Math.abs(num))); + const factor = Math.pow(10, sf - 1 - magnitude); + return Math.round(num * factor) / factor; + }; + + const existingRounded = roundToSignificantFigures(existingVal, 6); + const incomingRounded = roundToSignificantFigures(incomingVal, 6); + + if (existingRounded !== incomingRounded) { + console.log( + ` ${key}: existing=${existingVal} (${existingRounded} @ 6SF) vs incoming=${incomingVal} (${incomingRounded} @ 6SF)`, + ); + diffCount++; + } + } else if (existingVal !== incomingVal) { + console.log( + ` ${key}: existing="${existingVal}" (${typeof existingVal}) vs incoming="${incomingVal}" (${typeof incomingVal})`, + ); + diffCount++; + } + } + + console.log(`Total field differences: ${diffCount}`); + } + + return isEqual; +} + // Helper function to get the list of changed fields between two records function getChangedFields( existing: T, @@ -573,8 +749,9 @@ async function generateSummaryForGroup( const batchStartTime = performance.now(); // Prepare batch of unique keys - const batchUniqueKeys = batch.map((row) => { + const batchUniqueKeys = batch.map((row, rowIndex) => { const uniqueKeys: KeysMap = {}; + const globalRowIndex = batchStart + rowIndex; // Add keys from the current table and all ancestor tables for (const [key, value] of Object.entries(row)) { @@ -611,7 +788,9 @@ async function generateSummaryForGroup( // Process batch results batch.forEach((row, index) => { + const globalRowIndex = batchStart + index; const existing = existingRows[index]; + const uniqueKeys = batchUniqueKeys[index]; if (existing) { // Get the list of changed fields @@ -697,7 +876,7 @@ async function convertIngestedDataToSummary( ), ); - summaryData[tableNameTyped] = summary as any; + summaryData[tableNameTyped] = summary; } return summaryData; @@ -817,7 +996,7 @@ function parseAgsToIngest( `accepted: ${ingested.accepted.length}, rejected: ${ingested.rejected.length}`, ); - ingestGroupsData[dbName] = ingested as any; + ingestGroupsData[dbName] = ingested; } const end = performance.now(); From 95cf096d2a5b0cee693e5a7e0e7bb05650f45b90 Mon Sep 17 00:00:00 2001 From: James Holcombe Date: Sun, 3 Aug 2025 22:36:00 +0100 Subject: [PATCH 2/2] lint --- .../imports/detailed-changes-glide.tsx | 1 - packages/app/src/db/crud/data/index.ts | 10 +- packages/app/src/lib/imports/summary/data.ts | 179 +----------------- 3 files changed, 4 insertions(+), 186 deletions(-) diff --git a/packages/app/src/components/imports/detailed-changes-glide.tsx b/packages/app/src/components/imports/detailed-changes-glide.tsx index d47bd59a..cbfc137c 100644 --- a/packages/app/src/components/imports/detailed-changes-glide.tsx +++ b/packages/app/src/components/imports/detailed-changes-glide.tsx @@ -14,7 +14,6 @@ import { Item, } from "@glideapps/glide-data-grid"; import { formatCellFromRawData } from "../data/data-table-panel/cell-formatter"; -import { RawCellData } from "@/hooks/use-virtualized-query-data"; type Props = { projectId: string; diff --git a/packages/app/src/db/crud/data/index.ts b/packages/app/src/db/crud/data/index.ts index b330ddde..334e6a6f 100644 --- a/packages/app/src/db/crud/data/index.ts +++ b/packages/app/src/db/crud/data/index.ts @@ -369,7 +369,7 @@ export async function readDataRowsByUniqueKeys( } // Process each batch item's unique keys - const batchFilters = batchUniqueKeys.map((uniqueKeys, index) => { + const batchFilters = batchUniqueKeys.map((uniqueKeys) => { const itemFilters: ReturnType[] = []; // First handle parent table keys to ensure proper hierarchy @@ -401,18 +401,14 @@ export async function readDataRowsByUniqueKeys( return and(...itemFilters); }); - const queryStart = performance.now(); - // Execute query with filters const result = await query .where(and(eq(projectIdColumn, projectId), or(...batchFilters))) .execute(); - const queryTime = performance.now() - queryStart; - // Map results back to the input order with abbreviation lookups const resultMap = new Map( - result.map((row, index) => { + result.map((row) => { const processedRow = { ...row }; // Replace abbreviation IDs with their codes for display @@ -445,7 +441,7 @@ export async function readDataRowsByUniqueKeys( }), ); - const finalResults = batchUniqueKeys.map((uniqueKeys, index) => { + const finalResults = batchUniqueKeys.map((uniqueKeys) => { // Convert abbreviation IDs to codes for consistent key string generation const keyGenerationKeys = convertAbbrIdsToCodesForKeyGeneration( uniqueKeys, diff --git a/packages/app/src/lib/imports/summary/data.ts b/packages/app/src/lib/imports/summary/data.ts index 96568a15..61d2dc32 100644 --- a/packages/app/src/lib/imports/summary/data.ts +++ b/packages/app/src/lib/imports/summary/data.ts @@ -428,180 +428,6 @@ type KeysMap = { [key: string]: unknown; }; -// Helper function to compare records and determine if data has actually changed -function areRecordsEqual( - existing: T, - incoming: T, - excludeFields: string[] = ["id", "createdAt", "updatedAt"], - tableName?: string, - abbrMap?: Map>, -): boolean { - // Add parent ID fields to exclude list - const extendedExcludeFields = [ - ...excludeFields, - // Common parent ID patterns - ...Object.keys(existing as Record).filter( - (key) => key.endsWith("Id") && key !== "id", // Exclude parent relationship IDs but keep the main 'id' - ), - ]; - - const existingFiltered = Object.fromEntries( - Object.entries(existing as Record).filter( - ([key]) => !extendedExcludeFields.includes(key), - ), - ); - - const incomingFiltered = Object.fromEntries( - Object.entries(incoming as Record).filter( - ([key]) => !extendedExcludeFields.includes(key), - ), - ); - - // Convert abbreviation IDs to codes in existing record for fair comparison - const convertAbbrIdsToCodesInRecord = (record: Record) => { - const converted = { ...record }; - - if (abbrMap) { - for (const [column, codeMap] of abbrMap.entries()) { - const value = record[column]; - if (value !== null && value !== undefined) { - // Handle both string and number IDs - const valueStr = value.toString(); - // The abbrMap has structure: column -> Map(code -> id) - // We need to find the code for the given ID - for (const [code, id] of codeMap.entries()) { - if (id.toString() === valueStr) { - converted[column] = code; - break; - } - } - } - } - } - - return converted; - }; - - // Convert abbreviation IDs to codes in existing record - const existingWithCodes = convertAbbrIdsToCodesInRecord(existingFiltered); - - // Normalize null/undefined values for comparison - const normalizeValue = (value: unknown) => { - if (value === null || value === undefined || value === "") { - return null; - } - // Convert Date objects to ISO strings for consistent comparison - if (value instanceof Date) { - return value.toISOString(); - } - return value; - }; - - // Ensure both objects have the same set of fields for fair comparison - const getAllFieldsFromBothRecords = ( - obj1: Record, - obj2: Record, - ) => { - const allKeys = new Set([...Object.keys(obj1), ...Object.keys(obj2)]); - - const normalizeRecord = (obj: Record) => { - const normalized: Record = {}; - for (const key of allKeys) { - normalized[key] = normalizeValue(obj[key]); - } - return normalized; - }; - - return { - normalized1: normalizeRecord(obj1), - normalized2: normalizeRecord(obj2), - }; - }; - - const { normalized1: normalizedExisting, normalized2: normalizedIncoming } = - getAllFieldsFromBothRecords(existingWithCodes, incomingFiltered); - - // Use deep equality comparison for all values - const deepEqual = ( - obj1: Record, - obj2: Record, - ): boolean => { - const keys1 = Object.keys(obj1); - const keys2 = Object.keys(obj2); - - if (keys1.length !== keys2.length) { - return false; - } - - for (const key of keys1) { - if (!keys2.includes(key)) { - return false; - } - - const val1 = obj1[key]; - const val2 = obj2[key]; - - // Regular equality for all values - if (val1 !== val2) { - return false; - } - } - - return true; - }; - - const isEqual = deepEqual(normalizedExisting, normalizedIncoming); - - // DEBUG: Add targeted debugging for remaining location_details mismatches - if (!isEqual && tableName === "location_details") { - // Log all mismatches for location_details to identify the remaining 2 - console.log(`\nšŸ” DEBUG: location_details comparison mismatch:`); - console.log("Excluded fields:", extendedExcludeFields); - - // Find specific field differences - const allKeys = new Set([ - ...Object.keys(normalizedExisting), - ...Object.keys(normalizedIncoming), - ]); - - console.log("Field differences:"); - let diffCount = 0; - for (const key of allKeys) { - const existingVal = normalizedExisting[key]; - const incomingVal = normalizedIncoming[key]; - - // Special handling for numeric comparison debugging - if (typeof existingVal === "number" && typeof incomingVal === "number") { - const roundToSignificantFigures = (num: number, sf: number): number => { - if (num === 0) return 0; - const magnitude = Math.floor(Math.log10(Math.abs(num))); - const factor = Math.pow(10, sf - 1 - magnitude); - return Math.round(num * factor) / factor; - }; - - const existingRounded = roundToSignificantFigures(existingVal, 6); - const incomingRounded = roundToSignificantFigures(incomingVal, 6); - - if (existingRounded !== incomingRounded) { - console.log( - ` ${key}: existing=${existingVal} (${existingRounded} @ 6SF) vs incoming=${incomingVal} (${incomingRounded} @ 6SF)`, - ); - diffCount++; - } - } else if (existingVal !== incomingVal) { - console.log( - ` ${key}: existing="${existingVal}" (${typeof existingVal}) vs incoming="${incomingVal}" (${typeof incomingVal})`, - ); - diffCount++; - } - } - - console.log(`Total field differences: ${diffCount}`); - } - - return isEqual; -} - // Helper function to get the list of changed fields between two records function getChangedFields( existing: T, @@ -749,9 +575,8 @@ async function generateSummaryForGroup( const batchStartTime = performance.now(); // Prepare batch of unique keys - const batchUniqueKeys = batch.map((row, rowIndex) => { + const batchUniqueKeys = batch.map((row) => { const uniqueKeys: KeysMap = {}; - const globalRowIndex = batchStart + rowIndex; // Add keys from the current table and all ancestor tables for (const [key, value] of Object.entries(row)) { @@ -788,9 +613,7 @@ async function generateSummaryForGroup( // Process batch results batch.forEach((row, index) => { - const globalRowIndex = batchStart + index; const existing = existingRows[index]; - const uniqueKeys = batchUniqueKeys[index]; if (existing) { // Get the list of changed fields