From 7bbd8b17785c5a0f15c6fdf2bc40b292a7a97912 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:17:42 -0700 Subject: [PATCH] fix: Exclude out-of-range rollupLambda partitions Fixes #11317 --- .../PreAggregationPartitionRangeLoader.ts | 6 +- .../test/unit/PreAggregations.test.ts | 105 +++++++++++++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts index 9c63e26f0128f..690ebf3e603ff 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts @@ -274,7 +274,11 @@ export class PreAggregationPartitionRangeLoader { } const rollupLambdaResults = this.preAggregationsTablesToTempTables.filter(tempTableResult => tempTableResult[1].rollupLambdaId === this.preAggregation.rollupLambdaId); const filteredResults = loadResults.filter( - r => (this.preAggregation.lastRollupLambda || reformatInIsoLocal(r.buildRangeEnd) === reformatInIsoLocal(r.partitionRange[1])) && + r => (!this.preAggregation.matchedTimeDimensionDateRange || !!PreAggregationPartitionRangeLoader.intersectDateRanges( + r.partitionRange, + this.preAggregation.matchedTimeDimensionDateRange + )) && + (this.preAggregation.lastRollupLambda || reformatInIsoLocal(r.buildRangeEnd) === reformatInIsoLocal(r.partitionRange[1])) && rollupLambdaResults.every(result => !result[1].buildRangeEnd || reformatInIsoLocal(result[1].buildRangeEnd) < reformatInIsoLocal(r.partitionRange[0])) ); if (filteredResults.length === 0) { diff --git a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts index e63d6a54b8f31..5f7ab95db5a18 100644 --- a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts +++ b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts @@ -93,14 +93,18 @@ const mockPreAggregation = (overrides: Record = {}) => ({ ...overrides, }); -const createLoader = (overrides: Record = {}, options: Record = {}) => { +const createLoader = ( + overrides: Record = {}, + options: Record = {}, + preAggregationsTablesToTempTables: any[] = [], +) => { const loader = new PreAggregationPartitionRangeLoader( {} as any, // driverFactory {} as any, // logger { options: {} } as any, // queryCache {} as any, // preAggregations mockPreAggregation(overrides) as any, - [], // preAggregationsTablesToTempTables + preAggregationsTablesToTempTables, {} as any, // loadCache options as any, ); @@ -581,6 +585,103 @@ describe('PreAggregations', () => { }); }); + describe('rollupLambda partition filtering', () => { + let loadPreAggregationMock: jest.SpyInstance; + + beforeEach(() => { + loadPreAggregationMock = jest.spyOn(PreAggregationLoader.prototype, 'loadPreAggregation').mockResolvedValue({ + targetTableName: 'hot_partition', + refreshKeyValues: [], + lastUpdatedAt: 1, + buildRangeEnd: '2024-02-01T23:59:59.999', + }); + }); + + afterEach(() => { + loadPreAggregationMock.mockRestore(); + }); + + test('should return an empty result for an out-of-range last rollupLambda partition', async () => { + const loader = createLoader({ + rollupLambdaId: 'orders.lambda', + lastRollupLambda: true, + matchedTimeDimensionDateRange: ['2024-01-01T00:00:00.000', '2024-01-31T23:59:59.999'], + }); + jest.spyOn(loader as any, 'partitionRanges').mockResolvedValue({ + buildRange: ['2024-02-01T00:00:00.000', '2024-02-01T23:59:59.999'], + partitionRanges: [['2024-02-01T00:00:00.000', '2024-02-01T23:59:59.999']], + }); + + const result = await loader.loadPreAggregations(); + + expect(result.targetTableName).toBe('(SELECT * FROM hot_partition WHERE 1 = 0)'); + expect(result.buildRangeEnd).toBeFalsy(); + }); + + test('should retain a last rollupLambda partition that overlaps the requested range', async () => { + const loader = createLoader( + { + rollupLambdaId: 'orders.lambda', + lastRollupLambda: true, + matchedTimeDimensionDateRange: ['2024-01-30T12:00:00.000', '2024-01-31T23:59:59.999'], + }, + {}, + [[ + 'batch_rollup', + { + targetTableName: 'batch_partition', + refreshKeyValues: [], + lastUpdatedAt: 1, + buildRangeEnd: '2024-01-30T23:59:59.999', + rollupLambdaId: 'orders.lambda', + }, + ]], + ); + jest.spyOn(loader as any, 'partitionRanges').mockResolvedValue({ + buildRange: ['2024-01-31T00:00:00.000', '2024-02-01T23:59:59.999'], + partitionRanges: [['2024-01-31T00:00:00.000', '2024-02-01T23:59:59.999']], + }); + + const result = await loader.loadPreAggregations(); + + expect(result.targetTableName).toBe('hot_partition'); + expect(result.buildRangeEnd).toBe('2024-02-01T23:59:59.999'); + }); + + test('should retain existing rollupLambda filtering without a matched range', async () => { + const loader = createLoader({ + rollupLambdaId: 'orders.lambda', + lastRollupLambda: true, + }); + jest.spyOn(loader as any, 'partitionRanges').mockResolvedValue({ + buildRange: ['2024-02-01T00:00:00.000', '2024-02-01T23:59:59.999'], + partitionRanges: [['2024-02-01T00:00:00.000', '2024-02-01T23:59:59.999']], + }); + + const result = await loader.loadPreAggregations(); + + expect(result.targetTableName).toBe('hot_partition'); + expect(result.buildRangeEnd).toBe('2024-02-01T23:59:59.999'); + }); + + test('should retain a rollupLambda partition that touches the requested range boundary', async () => { + const loader = createLoader({ + rollupLambdaId: 'orders.lambda', + lastRollupLambda: true, + matchedTimeDimensionDateRange: ['2024-01-31T00:00:00.000', '2024-02-01T00:00:00.000'], + }); + jest.spyOn(loader as any, 'partitionRanges').mockResolvedValue({ + buildRange: ['2024-02-01T00:00:00.000', '2024-02-01T23:59:59.999'], + partitionRanges: [['2024-02-01T00:00:00.000', '2024-02-01T23:59:59.999']], + }); + + const result = await loader.loadPreAggregations(); + + expect(result.targetTableName).toBe('hot_partition'); + expect(result.buildRangeEnd).toBe('2024-02-01T23:59:59.999'); + }); + }); + describe('partitionTableName', () => { test('should generate correct table names for different granularities', () => { const testDateRange: [string, string] = ['2024-01-05T12:34:56.789', '2024-01-05T23:59:59.999'];