diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts index 4f28f1a642e3e..71eea71b64cf1 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts @@ -38,13 +38,19 @@ export function version(cacheKey) { const byte = digestBuffer.readUInt8(i); shiftCounter += 8; // eslint-disable-next-line operator-assignment,no-bitwise - residue = (byte << (shiftCounter - 8)) | residue; + // Use >>> 0 to ensure unsigned 32-bit integer after the OR operation, + // preventing negative values when high bits are set (byte >= 128 with shift >= 24) + // eslint-disable-next-line operator-assignment,no-bitwise + residue = ((byte << (shiftCounter - 8)) | residue) >>> 0; // eslint-disable-next-line no-bitwise - while (residue >> 5) { + // Use >>> (unsigned right shift) to ensure proper comparison and shifting + // of unsigned values, preventing infinite loops with negative residues + // eslint-disable-next-line operator-assignment,no-bitwise + while (residue >>> 5) { result += hashCharset.charAt(residue % 32); shiftCounter -= 5; // eslint-disable-next-line operator-assignment,no-bitwise - residue = residue >> 5; + residue = residue >>> 5; } } diff --git a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts similarity index 51% rename from packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js rename to packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts index d72d7e5492ad5..09858cbea2a43 100644 --- a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js +++ b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.ts @@ -1,5 +1,5 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable global-require */ -/* globals describe, jest, beforeEach, test, expect */ import R from 'ramda'; import { BUILD_RANGE_END_LOCAL, @@ -7,73 +7,75 @@ import { FROM_PARTITION_RANGE, TO_PARTITION_RANGE } from '@cubejs-backend/shared'; -import { PreAggregationPartitionRangeLoader, PreAggregations } from '../../src'; +import { PreAggregationPartitionRangeLoader, PreAggregations, version } from '../../src'; class MockDriver { - constructor() { - this.tables = []; - this.executedQueries = []; - this.cancelledQueries = []; - this.now = new Date().getTime(); - } + public tables: string[] = []; + + public executedQueries: string[] = []; + + public cancelledQueries: string[] = []; - query(query) { + public now: number = Date.now(); + + public schema: string | null = null; + + public query(query: string): Promise & { cancel?: () => Promise } { this.executedQueries.push(query); - let promise = Promise.resolve([query]); - if (query.match('orders_too_big')) { - promise = promise.then((res) => new Promise(resolve => setTimeout(() => resolve(res), 3000))); - } + const promise: Promise & { cancel?: () => Promise } = query.match('orders_too_big') + ? new Promise(resolve => setTimeout(() => resolve([query]), 3000)) + : Promise.resolve([query]); promise.cancel = async () => { this.cancelledQueries.push(query); }; return promise; } - async getTablesQuery(schema) { + public async getTablesQuery(schema: string) { return this.tables.map(t => ({ table_name: t.replace(`${schema}.`, '') })); } - async createSchemaIfNotExists(schema) { + public async createSchemaIfNotExists(schema: string) { this.schema = schema; return null; } - loadPreAggregationIntoTable(preAggregationTableName, loadSql) { + public loadPreAggregationIntoTable(preAggregationTableName: string, loadSql: string) { this.tables.push(preAggregationTableName.substring(0, 100)); return this.query(loadSql); } - async dropTable(tableName) { + public async dropTable(tableName: string) { this.tables = this.tables.filter(t => t !== tableName); return this.query(`DROP TABLE ${tableName}`); } - async downloadTable(table) { + public async downloadTable(table: string) { return { rows: await this.query(`SELECT * FROM ${table}`) }; } - async tableColumnTypes(_table) { + public async tableColumnTypes(_table: string) { return []; } - async uploadTable(table, columns, _tableData) { + public async uploadTable(table: string, columns: any, _tableData: any) { await this.createTable(table, columns); } - createTable(quotedTableName, _columns) { + public createTable(quotedTableName: string, _columns: any) { this.tables.push(quotedTableName); } - readOnly() { + public readOnly() { return false; } - nowTimestamp() { + public nowTimestamp() { return this.now; } } -const mockPreAggregation = (overrides = {}) => ({ +const mockPreAggregation = (overrides: Record = {}) => ({ tableName: 'test_table', partitionGranularity: 'day', timezone: 'UTC', @@ -91,19 +93,19 @@ const mockPreAggregation = (overrides = {}) => ({ ...overrides, }); -const createLoader = (overrides = {}, options = {}) => { +const createLoader = (overrides: Record = {}, options: Record = {}) => { const loader = new PreAggregationPartitionRangeLoader( - {}, // driverFactory - {}, // logger - { options: {} }, // queryCache - {}, // preAggregations - mockPreAggregation(overrides), + {} as any, // driverFactory + {} as any, // logger + { options: {} } as any, // queryCache + {} as any, // preAggregations + mockPreAggregation(overrides) as any, [], // preAggregationsTablesToTempTables - {}, // loadCache - options, + {} as any, // loadCache + options as any, ); - jest.spyOn(loader, 'loadRangeQuery').mockImplementation(async (query, _partitionRange) => { + jest.spyOn(loader as any, 'loadRangeQuery').mockImplementation(async (query: any, _partitionRange: any) => { if (query[0].includes('MIN')) { return [{ value: '2024-01-01T00:00:00.000' }]; } @@ -114,14 +116,14 @@ const createLoader = (overrides = {}, options = {}) => { }; describe('PreAggregations', () => { - let mockDriver = null; - let mockExternalDriver = null; - let mockDriverFactory = null; - let mockDriverReadOnlyFactory = null; - let mockExternalDriverFactory = null; - let queryCache = null; - - const basicQuery = { + let mockDriver: MockDriver | null = null; + let mockExternalDriver: MockDriver | null = null; + let mockDriverFactory: (() => Promise) | null = null; + let mockDriverReadOnlyFactory: (() => Promise) | null = null; + let mockExternalDriverFactory: (() => Promise) | null = null; + let queryCache: any = null; + + const basicQuery: any = { query: 'SELECT "orders__created_at_week" "orders__created_at_week", sum("orders__count") "orders__count" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20191101) as partition_union WHERE ("orders__created_at_week" >= ($1::timestamptz::timestamptz AT TIME ZONE \'UTC\') AND "orders__created_at_week" <= ($2::timestamptz::timestamptz AT TIME ZONE \'UTC\')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000', values: ['2019-11-01T00:00:00Z', '2019-11-30T23:59:59Z'], cacheKeyQueries: { @@ -153,24 +155,26 @@ describe('PreAggregations', () => { beforeEach(() => { mockDriver = new MockDriver(); mockExternalDriver = new MockDriver(); - mockDriverFactory = async () => mockDriver; + mockDriverFactory = async () => mockDriver!; mockDriverReadOnlyFactory = async () => { - const driver = mockDriver; + const driver = mockDriver!; jest.spyOn(driver, 'readOnly').mockImplementation(() => true); return driver; }; mockExternalDriverFactory = async () => { - const driver = mockExternalDriver; - driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1593709044209'); + const driver = mockExternalDriver!; + driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1593709044209', null); return driver; }; jest.resetModules(); + // Dynamic require after resetModules to ensure fresh module state + // eslint-disable-next-line @typescript-eslint/no-var-requires const { QueryCache } = require('../../src/orchestrator/QueryCache'); queryCache = new QueryCache( 'TEST', - mockDriverFactory, + mockDriverFactory as any, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, { @@ -184,15 +188,15 @@ describe('PreAggregations', () => { }); describe('loadAllPreAggregationsIfNeeded', () => { - let preAggregations = null; + let preAggregations: PreAggregations | null = null; beforeEach(async () => { preAggregations = new PreAggregations( 'TEST', - mockDriverFactory, + mockDriverFactory as any, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, - queryCache, + queryCache!, { queueOptions: () => ({ executionTimeout: 1, @@ -203,77 +207,77 @@ describe('PreAggregations', () => { }); test('synchronously create rollup from scratch', async () => { - mockDriver.now = 12345000; - const { preAggregationsTablesToTempTables: result } = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryWithRenew); + mockDriver!.now = 12345000; + const { preAggregationsTablesToTempTables: result } = await preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryWithRenew); expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/); expect(result[0][1].lastUpdatedAt).toEqual(12345000); }); }); describe('loadAllPreAggregationsIfNeeded with external rollup and writable source', () => { - let preAggregations = null; + let preAggregations: PreAggregations | null = null; beforeEach(async () => { preAggregations = new PreAggregations( 'TEST', - mockDriverFactory, + mockDriverFactory as any, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, - queryCache, + queryCache!, { queueOptions: () => ({ executionTimeout: 1, concurrency: 2, }), - externalDriverFactory: mockExternalDriverFactory, + externalDriverFactory: mockExternalDriverFactory as any, }, ); }); test('refresh external preaggregation with a writable source (refreshImplTempTableExternalStrategy)', async () => { - const { preAggregationsTablesToTempTables: result } = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal); + const { preAggregationsTablesToTempTables: result } = await preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryExternal); expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/); expect(result[0][1].lastUpdatedAt).toEqual(1593709044209); }); }); describe('loadAllPreAggregationsIfNeeded with external rollup and readonly source', () => { - let preAggregations = null; + let preAggregations: PreAggregations | null = null; beforeEach(async () => { preAggregations = new PreAggregations( 'TEST', - mockDriverReadOnlyFactory, + mockDriverReadOnlyFactory as any, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, - queryCache, + queryCache!, { queueOptions: () => ({ executionTimeout: 1, concurrency: 2, }), - externalDriverFactory: mockExternalDriverFactory, + externalDriverFactory: mockExternalDriverFactory as any, }, ); }); test('refresh external preaggregation with a writable source (refreshImplStreamExternalStrategy)', async () => { - const { preAggregationsTablesToTempTables: result } = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal); + const { preAggregationsTablesToTempTables: result } = await preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryExternal); expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/); expect(result[0][1].lastUpdatedAt).toEqual(1593709044209); }); }); describe('loadAllPreAggregationsIfNeeded with externalRefresh true', () => { - let preAggregations = null; + let preAggregations: PreAggregations | null = null; beforeEach(async () => { preAggregations = new PreAggregations( 'TEST', - mockDriverFactory, + mockDriverFactory as any, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, - queryCache, + queryCache!, { queueOptions: () => ({ executionTimeout: 1, @@ -285,18 +289,18 @@ describe('PreAggregations', () => { }); test('fail if waitForRenew is also specified', async () => { - await expect(preAggregations.loadAllPreAggregationsIfNeeded(basicQueryWithRenew)) + await expect(preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryWithRenew)) .rejects.toThrowError(/Invalid configuration/); }); test('fail if rollup doesn\'t already exist', async () => { - await expect(preAggregations.loadAllPreAggregationsIfNeeded(basicQuery)) + await expect(preAggregations!.loadAllPreAggregationsIfNeeded(basicQuery)) .rejects.toThrowError(/No pre-aggregation partitions were built yet/); }); }); describe('loadAllPreAggregationsIfNeeded with external rollup and externalRefresh true', () => { - let preAggregations = null; + let preAggregations: PreAggregations | null = null; beforeEach(async () => { preAggregations = new PreAggregations( @@ -304,49 +308,49 @@ describe('PreAggregations', () => { () => { throw new Error('The source database factory should never be called when externalRefresh is true, as it will trigger testConnection'); }, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, - queryCache, + queryCache!, { queueOptions: () => ({ executionTimeout: 1, concurrency: 2, }), - externalDriverFactory: mockExternalDriverFactory, + externalDriverFactory: mockExternalDriverFactory as any, externalRefresh: true, }, ); }); test('fail if waitForRenew is also specified', async () => { - await expect(preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternalWithRenew)) + await expect(preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryExternalWithRenew)) .rejects.toThrowError(/Invalid configuration/); }); test('load external preaggregation without communicating to the source database', async () => { - const { preAggregationsTablesToTempTables: result } = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal); + const { preAggregationsTablesToTempTables: result } = await preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryExternal); expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/); expect(result[0][1].lastUpdatedAt).toEqual(1593709044209); }); }); describe('naming_version tests', () => { - let preAggregations = null; + let preAggregations: PreAggregations | null = null; beforeEach(async () => { preAggregations = new PreAggregations( 'TEST', - mockDriverFactory, + mockDriverFactory as any, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, - queryCache, + queryCache!, { queueOptions: () => ({ executionTimeout: 1, concurrency: 2, }), externalDriverFactory: async () => { - const driver = mockExternalDriver; - driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1593709044209'); - driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652'); + const driver = mockExternalDriver!; + driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1593709044209', null); + driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652', null); return driver; }, }, @@ -373,31 +377,31 @@ describe('PreAggregations', () => { }); test('naming_version and sort by last_updated_at', async () => { - const { preAggregationsTablesToTempTables: result } = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal); + const { preAggregationsTablesToTempTables: result } = await preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryExternal); expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652/); expect(result[0][1].lastUpdatedAt).toEqual(1600329890000); }); }); describe('naming_version sort tests', () => { - let preAggregations = null; + let preAggregations: PreAggregations | null = null; beforeEach(async () => { preAggregations = new PreAggregations( 'TEST', - mockDriverFactory, + mockDriverFactory as any, // eslint-disable-next-line @typescript-eslint/no-empty-function () => {}, - queryCache, + queryCache!, { queueOptions: () => ({ executionTimeout: 1, concurrency: 2, }), externalDriverFactory: async () => { - const driver = mockExternalDriver; - driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1893709044209'); - driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652'); + const driver = mockExternalDriver!; + driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1893709044209', null); + driver.createTable('stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652', null); return driver; }, }, @@ -405,7 +409,7 @@ describe('PreAggregations', () => { }); test('naming_version and sort by last_updated_at', async () => { - const { preAggregationsTablesToTempTables: result } = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal); + const { preAggregationsTablesToTempTables: result } = await preAggregations!.loadAllPreAggregationsIfNeeded(basicQueryExternal); expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1893709044209/); expect(result[0][1].lastUpdatedAt).toEqual(1893709044209); }); @@ -466,12 +470,12 @@ describe('PreAggregations', () => { test('throws error if range is not a tuple of two strings', () => { expect(() => PreAggregationPartitionRangeLoader.intersectDateRanges( - ['2024-01-01T00:00:00.000'], + ['2024-01-01T00:00:00.000'] as any, ['2024-01-01T00:00:00.000', '2024-01-31T23:59:59.999'] )).toThrow('Date range expected to be an array with 2 elements'); expect(() => PreAggregationPartitionRangeLoader.intersectDateRanges( - ['2024-01-01T00:00:00.000', '2024-01-31T23:59:59.999', '2024-01-01T00:00:00.000'], + ['2024-01-01T00:00:00.000', '2024-01-31T23:59:59.999', '2024-01-01T00:00:00.000'] as any, ['2024-01-01T00:00:00.000', '2024-01-31T23:59:59.999'] )) .toThrow('Date range expected to be an array with 2 elements'); @@ -485,7 +489,7 @@ describe('PreAggregations', () => { describe('partitionTableName', () => { test('should generate correct table names for different granularities', () => { - const testDateRange = ['2024-01-05T12:34:56.789', '2024-01-05T23:59:59.999']; + const testDateRange: [string, string] = ['2024-01-05T12:34:56.789', '2024-01-05T23:59:59.999']; // Daily granularity expect(PreAggregationPartitionRangeLoader.partitionTableName( @@ -513,7 +517,7 @@ describe('PreAggregations', () => { describe('replaceQueryBuildRangeParams', () => { test('should replace BUILD_RANGE params with actual dates', async () => { const loader = createLoader(); - jest.spyOn(loader, 'loadBuildRange').mockResolvedValue([ + jest.spyOn(loader as any, 'loadBuildRange').mockResolvedValue([ '2023-01-01T00:00:00.000', '2023-01-31T23:59:59.999', ]); @@ -552,9 +556,9 @@ describe('PreAggregations', () => { expect(preAggDesc.tableName).toEqual('test_table'); expect(preAggDesc.buildRangeStart).toBeUndefined(); expect(preAggDesc.buildRangeEnd).toBeUndefined(); - expect(preAggDesc.loadSql[0].includes('test_table')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual(FROM_PARTITION_RANGE); - expect(preAggDesc.loadSql[1][1]).toEqual(TO_PARTITION_RANGE); + expect((preAggDesc.loadSql as any)[0].includes('test_table')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual(FROM_PARTITION_RANGE); + expect((preAggDesc.loadSql as any)[1][1]).toEqual(TO_PARTITION_RANGE); expect(preAggDesc.structureVersionLoadSql).toBeUndefined(); }); }); @@ -572,34 +576,34 @@ describe('PreAggregations', () => { expect(preAggDesc.tableName).toEqual('test_table20240101'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T23:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240101')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T23:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240101')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T23:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240101')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T23:59:59.999'); [, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table20240102'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-02T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-02T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240102')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-02T00:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-02T23:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240102')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-02T00:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T23:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240102')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-02T00:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-02T23:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240102')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-02T00:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-02T23:59:59.999'); [,, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table20240103'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240103')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T00:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240103')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T00:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240103')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-03T00:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240103')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-03T00:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); }); test('should construct correct partitionPreAggregations for dateRange in America/New_York (Day partitions)', async () => { @@ -614,34 +618,34 @@ describe('PreAggregations', () => { expect(preAggDesc.tableName).toEqual('test_table20231231'); expect(preAggDesc.buildRangeStart).toEqual('2023-12-31T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2023-12-31T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20231231')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T05:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T04:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20231231')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T05:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T04:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20231231')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2023-12-31T05:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T04:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20231231')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2023-12-31T05:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T04:59:59.999'); [, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table20240101'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-02T04:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240101')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T04:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240101')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T05:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-02T04:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240101')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T05:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-02T04:59:59.999'); [,,, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table20240103'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240103')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T05:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); // Because DateRangeEnd Mock Query returns it - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240103')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T05:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240103')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-03T05:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-04T04:59:59.999'); // Because DateRangeEnd Mock Query returns it + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240103')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-03T05:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-04T04:59:59.999'); }); test('should construct correct partitionPreAggregations for dateRange in Asia/Tokyo (Day partitions)', async () => { @@ -656,34 +660,34 @@ describe('PreAggregations', () => { expect(preAggDesc.tableName).toEqual('test_table20240101'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T15:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T14:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240101')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T15:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T14:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240101')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2023-12-31T15:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T14:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240101')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2023-12-31T15:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T14:59:59.999'); [, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table20240102'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-02T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-02T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240102')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T15:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-02T14:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240102')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T15:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T14:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240102')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T15:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-02T14:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240102')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T15:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-02T14:59:59.999'); [,,, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table20240104'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-04T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-04T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240104')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T15:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T14:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240104')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T15:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T14:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table20240104')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-03T15:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-04T14:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table20240104')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-03T15:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-04T14:59:59.999'); }); test('should construct correct partitionPreAggregations for dateRange in UTC (Hour partitions)', async () => { @@ -699,35 +703,35 @@ describe('PreAggregations', () => { expect(preAggDesc.tableName).toEqual('test_table2024010100'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T00:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010100')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010100')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2024010100')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2024010100')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T00:59:59.999'); [, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table2024010101'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T01:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T01:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010101')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010101')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2024010101')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2024010101')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T01:59:59.999'); // eslint-disable-next-line prefer-destructuring preAggDesc = results[71]; expect(preAggDesc.tableName).toEqual('test_table2024010323'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T23:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010323')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010323')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2024010323')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2024010323')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); }); test('should construct correct partitionPreAggregations for dateRange in America/New_York (Hour partitions)', async () => { @@ -743,35 +747,35 @@ describe('PreAggregations', () => { expect(preAggDesc.tableName).toEqual('test_table2023123119'); expect(preAggDesc.buildRangeStart).toEqual('2023-12-31T19:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2023-12-31T19:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2023123119')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2023123119')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2023123119')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2023123119')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T00:59:59.999'); [, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table2023123120'); expect(preAggDesc.buildRangeStart).toEqual('2023-12-31T20:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2023-12-31T20:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2023123120')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2023123120')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2023123120')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2023123120')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T01:59:59.999'); // eslint-disable-next-line prefer-destructuring preAggDesc = results[71]; expect(preAggDesc.tableName).toEqual('test_table2024010318'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T18:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T18:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010318')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010318')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2024010318')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2024010318')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); }); test('should construct correct partitionPreAggregations for dateRange in Asia/Tokyo (Hour partitions)', async () => { @@ -787,41 +791,41 @@ describe('PreAggregations', () => { expect(preAggDesc.tableName).toEqual('test_table2024010109'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T09:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T09:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010109')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010109')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2024010109')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2024010109')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T00:59:59.999'); [, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table2024010110'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T10:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T10:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010110')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010110')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2024010110')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2024010110')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-01T01:59:59.999'); // eslint-disable-next-line prefer-destructuring preAggDesc = results[71]; expect(preAggDesc.tableName).toEqual('test_table2024010408'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-04T08:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-04T08:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010408')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010408')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.loadSql as any)[0].includes('test_table2024010408')).toBeTruthy(); + expect((preAggDesc.loadSql as any)[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect((preAggDesc.loadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect((preAggDesc.structureVersionLoadSql as any)[0].includes('test_table2024010408')).toBeTruthy(); + expect((preAggDesc.structureVersionLoadSql as any)[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect((preAggDesc.structureVersionLoadSql as any)[1][1]).toEqual('2024-01-03T23:59:59.999'); }); }); describe('partitionPreAggregations', () => { test('should generate partitioned pre-aggregations', async () => { - const compilerCacheFn = jest.fn((subKey, fn) => fn()); + const compilerCacheFn = jest.fn((_subKey: any, fn: () => any) => fn()); const loader = createLoader( { partitionGranularity: 'day', @@ -830,7 +834,7 @@ describe('PreAggregations', () => { { compilerCacheFn } ); - jest.spyOn(loader, 'partitionRanges').mockResolvedValue({ + jest.spyOn(loader as any, 'partitionRanges').mockResolvedValue({ buildRange: ['2023-01-01T00:00:00.000', '2023-01-02T23:59:59.999'], partitionRanges: [ ['2023-01-01T00:00:00.000', '2023-01-01T23:59:59.999'], @@ -849,4 +853,149 @@ describe('PreAggregations', () => { ); }); }); + + describe('version function', () => { + test('should return a valid version string for simple input', () => { + const result = version(['test']); + expect(result).toBeTruthy(); + expect(typeof result).toBe('string'); + expect(result.length).toBe(8); + }); + + test('should not hang on complex cache keys with nested objects and arrays', () => { + // This test case previously caused an infinite loop due to signed bitwise operations + const complexCacheKey = [ + [ + "CREATE TABLE prod_pre_aggregations_mxc.m_x_c_actionable_hourly_agg_main_with_index_month120260112 AS SELECT\n `tags`.`description` `m_x_c_actionable_hourly_agg__description`, `tags`.`deviceName` `m_x_c_actionable_hourly_agg__device_name`, `tags`.`tagName` `m_x_c_actionable_hourly_agg__tag_name`, date_trunc('hour', from_utc_timestamp(`m_x_c_actionable_hourly_agg`.timestamp, 'America/Los_Angeles')) `m_x_c_actionable_hourly_agg__timestamp_hour`, sum(`m_x_c_actionable_hourly_agg`.`avgValue`) `m_x_c_actionable_hourly_agg__avg_value`, sum(`m_x_c_actionable_hourly_agg`.`firstValue`) `m_x_c_actionable_hourly_agg__first_value`, sum(`m_x_c_actionable_hourly_agg`.`lastValue`) `m_x_c_actionable_hourly_agg__last_value`, sum(`m_x_c_actionable_hourly_agg`.`maxValue`) `m_x_c_actionable_hourly_agg__max_value`, sum(`m_x_c_actionable_hourly_agg`.`minValue`) `m_x_c_actionable_hourly_agg__min_value`, sum(`m_x_c_actionable_hourly_agg`.`modeValue`) `m_x_c_actionable_hourly_agg__mode_value`\n FROM\n (SELECT *,\n LAST(lastValue) OVER(PARTITION BY DEVICETAG ORDER BY timestamp ASC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS lastValue2 \n FROM prodcatalog.litmus.mxc_litmus_agg_zorder_action_hour\n ) AS `m_x_c_actionable_hourly_agg`\nLEFT JOIN prodcatalog.litmus.mxc_litmus_agg_tagt AS `tags` ON `m_x_c_actionable_hourly_agg`.`DEVICETAG` = `tags`.`DEVICETAG` WHERE (`m_x_c_actionable_hourly_agg`.timestamp >= from_utc_timestamp(replace(replace(?, 'T', ' '), 'Z', ''), 'UTC') AND `m_x_c_actionable_hourly_agg`.timestamp <= from_utc_timestamp(replace(replace(?, 'T', ' '), 'Z', ''), 'UTC')) GROUP BY 1, 2, 3, 4", + [ + '2026-01-12T08:00:00.000Z', + '2026-01-19T07:59:59.999Z' + ], + {} + ], + [ + { + indexName: 'm_x_c_actionable_hourly_agg_main_with_index_month1_device_tag_description_index', + sql: [ + "CREATE INDEX m_x_c_actionable_hourly_agg_main_with_index_month1_device_tag_description_index ON prod_pre_aggregations_mxc.m_x_c_actionable_hourly_agg_main_with_index_month120260112 (`m_x_c_actionable_hourly_agg__device_name`, `m_x_c_actionable_hourly_agg__tag_name`, `m_x_c_actionable_hourly_agg__description`, `m_x_c_actionable_hourly_agg__timestamp_hour`)", + [], + {} + ] + }, + { + indexName: 'm_x_c_actionable_hourly_agg_main_with_index_month1_tag_description_device_index', + sql: [ + "CREATE INDEX m_x_c_actionable_hourly_agg_main_with_index_month1_tag_description_device_index ON prod_pre_aggregations_mxc.m_x_c_actionable_hourly_agg_main_with_index_month120260112 (`m_x_c_actionable_hourly_agg__tag_name`, `m_x_c_actionable_hourly_agg__description`, `m_x_c_actionable_hourly_agg__device_name`)", + [], + {} + ] + } + ], + [ + [ + { + refresh_key: null + } + ] + ] + ]; + + // The function should complete without hanging (timeout will fail the test if it hangs) + const result = version(complexCacheKey); + expect(result).toBeTruthy(); + expect(typeof result).toBe('string'); + expect(result.length).toBe(8); + }); + + test('should handle inputs that produce high byte values in MD5 digest', () => { + // Test various inputs to ensure unsigned bit operations work correctly + const testCases = [ + 'test', + { key: 'value' }, + [1, 2, 3], + 'a'.repeat(1000), + { nested: { deep: { value: 'test' } } }, + ]; + + for (const input of testCases) { + const result = version(input); + expect(result).toBeTruthy(); + expect(typeof result).toBe('string'); + expect(result.length).toBe(8); + // Verify the result only contains valid charset characters + expect(result).toMatch(/^[a-z0-5]+$/); + } + }); + + test('should produce same results as old implementation for backward compatibility', () => { + // Old implementation (before the unsigned shift fix) + // This would hang on certain inputs, but for inputs that don't trigger the bug, + // it should produce the same results as the new implementation + const crypto = require('crypto'); + + function oldVersion(cacheKey: any): string | null { + let result = ''; + + const hashCharset = 'abcdefghijklmnopqrstuvwxyz012345'; + const digestBuffer = crypto.createHash('md5').update(JSON.stringify(cacheKey)).digest(); + + let residue = 0; + let shiftCounter = 0; + + for (let i = 0; i < 5; i++) { + const byte = digestBuffer.readUInt8(i); + shiftCounter += 8; + // eslint-disable-next-line operator-assignment,no-bitwise + residue = (byte << (shiftCounter - 8)) | residue; + // eslint-disable-next-line no-bitwise + while (residue >> 5) { + result += hashCharset.charAt(residue % 32); + shiftCounter -= 5; + // eslint-disable-next-line operator-assignment,no-bitwise + residue = residue >> 5; + } + } + + result += hashCharset.charAt(residue % 32); + + return result; + } + + // 20 hard-coded test cases with their expected version strings + // These are keys that work correctly with both old and new implementations + const testCases: Array<{ key: any; expected: string }> = [ + { key: 'simple_string', expected: 'lyidb3bl' }, + { key: 'hello_world', expected: 'sz1y5yvi' }, + { key: 'test_key_123', expected: 'tpsualal' }, + { key: ['array', 'of', 'strings'], expected: 'sbll5p55' }, + { key: { name: 'object', value: 42 }, expected: 'sq5wacbz' }, + { key: [1, 2, 3, 4, 5], expected: 'sercayat' }, + { key: { nested: { level: 2 } }, expected: '5hdmsxe4' }, + { key: 'SELECT * FROM users', expected: 'bzasp2ee' }, + { key: ['CREATE TABLE test', ['param1', 'param2']], expected: 'ghze1maw' }, + { key: { sql: 'SELECT 1', params: [] }, expected: 'crhopprj' }, + { key: 'pre_aggregation_key_v1', expected: 'ldkocgfh' }, + { key: ['2024-01-01', '2024-12-31'], expected: 'oojrcwo3' }, + { key: { timezone: 'UTC', granularity: 'day' }, expected: 'es2subt' }, + { key: 'cube_query_cache_key', expected: 'zxeekgd0' }, + { key: [{ id: 1 }, { id: 2 }, { id: 3 }], expected: 'kxoosnjv' }, + { key: { dimensions: ['a', 'b'], measures: ['c'] }, expected: '1ppe4o4c' }, + { key: 'abcdefghijklmnopqrstuvwxyz', expected: 'aj4ij4kb' }, + { key: '0123456789', expected: 'wsidmvgj' }, + { key: { empty: {}, arr: [] }, expected: 'jvhxdtaj' }, + { key: ['mixed', 123, true, null, { x: 'y' }], expected: 'qzfgu32u' }, + ]; + + for (const { key, expected } of testCases) { + // Verify new implementation matches expected + const newResult = version(key); + expect(newResult).toBe(expected); + + // Verify old implementation also matches (proving backward compatibility) + const oldResult = oldVersion(key); + expect(oldResult).not.toBeNull(); // Should not hang + expect(oldResult).toBe(expected); + } + }); + }); });