Skip to content

Commit 1e7f9bb

Browse files
pulpdrewpulpdrew
committed
feat: Optimize and fix filtering on toStartOfX primary key expressions
1 parent 43e32aa commit 1e7f9bb

File tree

5 files changed

+492
-17
lines changed

5 files changed

+492
-17
lines changed

‎.changeset/fluffy-mails-sparkle.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@hyperdx/common-utils": patch
3+
"@hyperdx/app": patch
4+
---
5+
6+
feat: Optimize and fix filtering on toStartOfX primary key expressions

‎packages/common-utils/src/__tests__/renderChartConfig.test.ts‎

Lines changed: 168 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
import { chSql, parameterizedQueryToSql } from '@/clickhouse';
1+
import { chSql, ColumnMeta, parameterizedQueryToSql } from '@/clickhouse';
22
import { Metadata } from '@/metadata';
33
import {
44
ChartConfigWithOptDateRange,
55
DisplayType,
66
MetricsDataType,
77
} from '@/types';
88

9-
import { renderChartConfig } from '../renderChartConfig';
9+
import { renderChartConfig, timeFilterExpr } from '../renderChartConfig';
1010

1111
describe('renderChartConfig', () => {
12-
let mockMetadata: Metadata;
12+
let mockMetadata: jest.Mocked<Metadata>;
1313

1414
beforeEach(() => {
1515
mockMetadata = {
@@ -19,7 +19,7 @@ describe('renderChartConfig', () => {
1919
]),
2020
getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null),
2121
getColumn: jest.fn().mockResolvedValue({ type: 'DateTime' }),
22-
} as unknown as Metadata;
22+
} as unknown as jest.Mocked<Metadata>;
2323
});
2424

2525
const gaugeConfiguration: ChartConfigWithOptDateRange = {
@@ -630,4 +630,168 @@ describe('renderChartConfig', () => {
630630
expect(actual).toMatchSnapshot();
631631
});
632632
});
633+
634+
describe('timeFilterExpr', () => {
635+
type TimeFilterExprTestCase = {
636+
timestampValueExpression: string;
637+
dateRangeStartInclusive?: boolean;
638+
dateRangeEndInclusive?: boolean;
639+
dateRange: [Date, Date];
640+
includedDataInterval?: string;
641+
expected: string;
642+
description: string;
643+
};
644+
645+
const testCases: TimeFilterExprTestCase[] = [
646+
{
647+
description: 'with basic timestampValueExpression',
648+
timestampValueExpression: 'timestamp',
649+
dateRange: [
650+
new Date('2025-02-12 00:12:34Z'),
651+
new Date('2025-02-14 00:12:34Z'),
652+
],
653+
expected: `(timestamp >= fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}) AND timestamp <= fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}))`,
654+
},
655+
{
656+
description: 'with dateRangeEndInclusive=false',
657+
timestampValueExpression: 'timestamp',
658+
dateRange: [
659+
new Date('2025-02-12 00:12:34Z'),
660+
new Date('2025-02-14 00:12:34Z'),
661+
],
662+
dateRangeEndInclusive: false,
663+
expected: `(timestamp >= fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}) AND timestamp < fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}))`,
664+
},
665+
{
666+
description: 'with dateRangeStartInclusive=false',
667+
timestampValueExpression: 'timestamp',
668+
dateRange: [
669+
new Date('2025-02-12 00:12:34Z'),
670+
new Date('2025-02-14 00:12:34Z'),
671+
],
672+
dateRangeStartInclusive: false,
673+
expected: `(timestamp > fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}) AND timestamp <= fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}))`,
674+
},
675+
{
676+
description: 'with includedDataInterval',
677+
timestampValueExpression: 'timestamp',
678+
dateRange: [
679+
new Date('2025-02-12 00:12:34Z'),
680+
new Date('2025-02-14 00:12:34Z'),
681+
],
682+
includedDataInterval: '1 WEEK',
683+
expected: `(timestamp >= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}), INTERVAL 1 WEEK) - INTERVAL 1 WEEK AND timestamp <= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}), INTERVAL 1 WEEK) + INTERVAL 1 WEEK)`,
684+
},
685+
{
686+
description: 'with date type timestampValueExpression',
687+
timestampValueExpression: 'date',
688+
dateRange: [
689+
new Date('2025-02-12 00:12:34Z'),
690+
new Date('2025-02-14 00:12:34Z'),
691+
],
692+
expected: `(date >= toDate(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()})) AND date <= toDate(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()})))`,
693+
},
694+
{
695+
description: 'with multiple timestampValueExpression parts',
696+
timestampValueExpression: 'timestamp, date',
697+
dateRange: [
698+
new Date('2025-02-12 00:12:34Z'),
699+
new Date('2025-02-14 00:12:34Z'),
700+
],
701+
expected: `(timestamp >= fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}) AND timestamp <= fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}))AND(date >= toDate(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()})) AND date <= toDate(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()})))`,
702+
},
703+
{
704+
description: 'with toStartOfDay() in timestampExpr',
705+
timestampValueExpression: 'toStartOfDay(timestamp)',
706+
dateRange: [
707+
new Date('2025-02-12 00:12:34Z'),
708+
new Date('2025-02-14 00:12:34Z'),
709+
],
710+
expected: `(toStartOfDay(timestamp) >= toStartOfDay(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()})) AND toStartOfDay(timestamp) <= toStartOfDay(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()})))`,
711+
},
712+
{
713+
description: 'with toStartOfDay () in timestampExpr',
714+
timestampValueExpression: 'toStartOfDay (timestamp)',
715+
dateRange: [
716+
new Date('2025-02-12 00:12:34Z'),
717+
new Date('2025-02-14 00:12:34Z'),
718+
],
719+
expected: `(toStartOfDay (timestamp) >= toStartOfDay(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()})) AND toStartOfDay (timestamp) <= toStartOfDay(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()})))`,
720+
},
721+
{
722+
description: 'with toStartOfInterval() in timestampExpr',
723+
timestampValueExpression:
724+
'toStartOfInterval(timestamp, INTERVAL 12 MINUTE)',
725+
dateRange: [
726+
new Date('2025-02-12 00:12:34Z'),
727+
new Date('2025-02-14 00:12:34Z'),
728+
],
729+
expected: `(toStartOfInterval(timestamp, INTERVAL 12 MINUTE) >= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}), INTERVAL 12 MINUTE) AND toStartOfInterval(timestamp, INTERVAL 12 MINUTE) <= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}), INTERVAL 12 MINUTE))`,
730+
},
731+
{
732+
description:
733+
'with toStartOfInterval() with lowercase interval in timestampExpr',
734+
timestampValueExpression:
735+
'toStartOfInterval(timestamp, interval 1 minute)',
736+
dateRange: [
737+
new Date('2025-02-12 00:12:34Z'),
738+
new Date('2025-02-14 00:12:34Z'),
739+
],
740+
expected: `(toStartOfInterval(timestamp, interval 1 minute) >= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}), interval 1 minute) AND toStartOfInterval(timestamp, interval 1 minute) <= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}), interval 1 minute))`,
741+
},
742+
{
743+
description: 'with toStartOfInterval() with timezone and offset',
744+
timestampValueExpression: `toStartOfInterval(timestamp, INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'), 'America/New_York')`,
745+
dateRange: [
746+
new Date('2025-02-12 00:12:34Z'),
747+
new Date('2025-02-14 00:12:34Z'),
748+
],
749+
expected: `(toStartOfInterval(timestamp, INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'), 'America/New_York') >= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'), 'America/New_York') AND toStartOfInterval(timestamp, INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'), 'America/New_York') <= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}), INTERVAL 1 MINUTE, toDateTime('2023-01-01 14:35:30'), 'America/New_York'))`,
750+
},
751+
{
752+
description: 'with nonstandard spacing',
753+
timestampValueExpression: ` toStartOfInterval ( timestamp , INTERVAL 1 MINUTE , toDateTime ( '2023-01-01 14:35:30' ), 'America/New_York' ) `,
754+
dateRange: [
755+
new Date('2025-02-12 00:12:34Z'),
756+
new Date('2025-02-14 00:12:34Z'),
757+
],
758+
expected: `(toStartOfInterval ( timestamp , INTERVAL 1 MINUTE , toDateTime ( '2023-01-01 14:35:30' ), 'America/New_York' ) >= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 00:12:34Z').getTime()}), INTERVAL 1 MINUTE, toDateTime ( '2023-01-01 14:35:30' ), 'America/New_York') AND toStartOfInterval ( timestamp , INTERVAL 1 MINUTE , toDateTime ( '2023-01-01 14:35:30' ), 'America/New_York' ) <= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-14 00:12:34Z').getTime()}), INTERVAL 1 MINUTE, toDateTime ( '2023-01-01 14:35:30' ), 'America/New_York'))`,
759+
},
760+
];
761+
762+
beforeEach(() => {
763+
mockMetadata.getColumn.mockImplementation(async ({ column }) =>
764+
column === 'date'
765+
? ({ type: 'Date' } as ColumnMeta)
766+
: ({ type: 'DateTime' } as ColumnMeta),
767+
);
768+
});
769+
770+
it.each(testCases)(
771+
'should generate a time filter expression $description',
772+
async ({
773+
timestampValueExpression,
774+
dateRangeEndInclusive = true,
775+
dateRangeStartInclusive = true,
776+
dateRange,
777+
expected,
778+
includedDataInterval,
779+
}) => {
780+
const actual = await timeFilterExpr({
781+
timestampValueExpression,
782+
dateRangeEndInclusive,
783+
dateRangeStartInclusive,
784+
dateRange,
785+
connectionId: 'test-connection',
786+
databaseName: 'default',
787+
metadata: mockMetadata,
788+
tableName: '',
789+
includedDataInterval,
790+
});
791+
792+
const actualSql = parameterizedQueryToSql(actual);
793+
expect(actualSql).toBe(expected);
794+
},
795+
);
796+
});
633797
});

0 commit comments

Comments
 (0)