From e41440597cbb232955cbcfd8e63ca92ebff150bd Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 7 Oct 2025 12:22:48 -0400 Subject: [PATCH 1/6] feat: Implement query chunking for charts --- .changeset/soft-donkeys-fetch.md | 5 + packages/app/package.json | 2 +- packages/app/src/components/DBTimeChart.tsx | 5 +- packages/app/src/components/PatternTable.tsx | 12 +- .../src/components/SearchTotalCountChart.tsx | 3 + .../hooks/__tests__/useChartConfig.test.tsx | 940 ++++++++++++++++++ packages/app/src/hooks/useChartConfig.tsx | 227 ++++- .../app/src/hooks/useOffsetPaginatedQuery.tsx | 79 +- packages/app/src/hooks/usePatterns.tsx | 15 +- packages/app/src/utils/searchWindows.ts | 79 ++ .../common-utils/src/renderChartConfig.ts | 25 +- yarn.lock | 20 +- 12 files changed, 1277 insertions(+), 135 deletions(-) create mode 100644 .changeset/soft-donkeys-fetch.md create mode 100644 packages/app/src/hooks/__tests__/useChartConfig.test.tsx create mode 100644 packages/app/src/utils/searchWindows.ts diff --git a/.changeset/soft-donkeys-fetch.md b/.changeset/soft-donkeys-fetch.md new file mode 100644 index 000000000..f9636d604 --- /dev/null +++ b/.changeset/soft-donkeys-fetch.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat: Implement query chunking for charts diff --git a/packages/app/package.json b/packages/app/package.json index cd2086661..4962530f5 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -41,7 +41,7 @@ "@mantine/spotlight": "7.9.2", "@microsoft/fetch-event-source": "^2.0.1", "@tabler/icons-react": "^3.5.0", - "@tanstack/react-query": "^5.56.2", + "@tanstack/react-query": "^5.90.2", "@tanstack/react-query-devtools": "^5.56.2", "@tanstack/react-table": "^8.7.9", "@tanstack/react-virtual": "^3.0.1", diff --git a/packages/app/src/components/DBTimeChart.tsx b/packages/app/src/components/DBTimeChart.tsx index b8fa4e4b4..dec94fc3f 100644 --- a/packages/app/src/components/DBTimeChart.tsx +++ b/packages/app/src/components/DBTimeChart.tsx @@ -62,7 +62,7 @@ function DBTimeChartComponent({ limit: { limit: 100000 }, }; - const { data, isLoading, isError, error, isPlaceholderData, isSuccess } = + const { data, isLoading, isError, error, isSuccess, isFetching } = useQueriedChartConfig(queriedConfig, { placeholderData: (prev: any) => prev, queryKey: [queryKeyPrefix, queriedConfig], @@ -75,7 +75,6 @@ function DBTimeChartComponent({ } }, [isError, isErrorExpanded, errorExpansion]); - const isLoadingOrPlaceholder = isLoading || isPlaceholderData; const { data: source } = useSource({ id: sourceId }); const { graphResults, timestampColumn, groupKeys, lineNames, lineColors } = @@ -338,7 +337,7 @@ function DBTimeChartComponent({ graphResults={graphResults} groupKeys={groupKeys} isClickActive={false} - isLoading={isLoadingOrPlaceholder} + isLoading={isFetching} lineColors={lineColors} lineNames={lineNames} logReferenceTimestamp={logReferenceTimestamp} diff --git a/packages/app/src/components/PatternTable.tsx b/packages/app/src/components/PatternTable.tsx index 6c82996d2..c44b4a7d9 100644 --- a/packages/app/src/components/PatternTable.tsx +++ b/packages/app/src/components/PatternTable.tsx @@ -29,10 +29,11 @@ export default function PatternTable({ const [selectedPattern, setSelectedPattern] = useState(null); - const { totalCount, isLoading: isTotalCountLoading } = useSearchTotalCount( - totalCountConfig, - totalCountQueryKeyPrefix, - ); + const { + totalCount, + isLoading: isTotalCountLoading, + isTotalCountComplete, + } = useSearchTotalCount(totalCountConfig, totalCountQueryKeyPrefix); const { data: groupedResults, @@ -46,7 +47,8 @@ export default function PatternTable({ totalCount, }); - const isLoading = isTotalCountLoading || isGroupedPatternsLoading; + const isLoading = + isTotalCountLoading || !isTotalCountComplete || isGroupedPatternsLoading; const sortedGroupedResults = useMemo(() => { return Object.values(groupedResults).sort( diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index b8b197802..a6da697c8 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -28,6 +28,8 @@ export function useSearchTotalCount( placeholderData: keepPreviousData, // no need to flash loading state when in live tail }); + const isTotalCountComplete = !!totalCountData?.isComplete; + const totalCount = useMemo(() => { return totalCountData?.data?.reduce( (p: number, v: any) => p + Number.parseInt(v['count()']), @@ -39,6 +41,7 @@ export function useSearchTotalCount( totalCount, isLoading, isError, + isTotalCountComplete, }; } diff --git a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx new file mode 100644 index 000000000..e4cead466 --- /dev/null +++ b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx @@ -0,0 +1,940 @@ +import React from 'react'; +import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import { + ChartConfigWithDateRange, + ChartConfigWithOptDateRange, + MetricsDataType, +} from '@hyperdx/common-utils/dist/types'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { renderHook, waitFor } from '@testing-library/react'; + +import { useClickhouseClient } from '@/clickhouse'; + +import { + getGranularityAlignedTimeWindows, + useQueriedChartConfig, +} from '../useChartConfig'; + +// Mock the clickhouse module +jest.mock('@/clickhouse', () => ({ + useClickhouseClient: jest.fn(), +})); + +// Mock the metadata module +jest.mock('@/metadata', () => ({ + getMetadata: jest.fn(() => ({ + sources: [], + connections: {}, + })), +})); + +// Mock the config module +jest.mock('@/config', () => ({ + IS_MTVIEWS_ENABLED: false, +})); + +// Create a mock ChartConfig +const createMockChartConfig = ( + overrides: Partial = {}, +): ChartConfigWithOptDateRange => + ({ + connection: 'foo', + from: { + databaseName: 'default', + tableName: 'otel_logs', + }, + where: '', + select: [{ aggCondition: '', aggFn: 'count', valueExpression: '' }], + timestampValueExpression: 'TimestampTime', + groupBy: 'SeverityText', + ...overrides, + }) as ChartConfigWithOptDateRange; + +const createMockQueryResponse = (data: any[]): ResponseJSON => { + return { + data, + rows: data.length, + meta: [ + { + name: 'count()', + type: 'UInt64', + }, + { + name: 'SeverityText', + type: 'LowCardinality(String)', + }, + { + name: '__hdx_time_bucket', + type: 'DateTime', + }, + ], + }; +}; + +describe('useChartConfig', () => { + describe('getGranularityAlignedTimeWindows', () => { + it('returns windows aligned to the granularity if the granularity is auto', () => { + expect( + getGranularityAlignedTimeWindows( + { + dateRange: [ + new Date('2023-01-10 00:00:00'), + new Date('2023-01-10 01:00:00'), + ], + granularity: 'auto', // will be 1 minute + timestampValueExpression: 'TimestampTime', + } as ChartConfigWithDateRange & { granularity: string }, + [ + 30, // 30s + 5 * 60, // 5m + 60 * 60, // 1hr + ], + ), + ).toEqual([ + { + dateRange: [ + new Date('2023-01-10 00:59:00'), // Aligned to minute, the auto-inferred granularity + new Date('2023-01-10 01:00:00'), + ], + dateRangeEndInclusive: undefined, + }, + { + dateRange: [ + new Date('2023-01-10 00:54:00'), + new Date('2023-01-10 00:59:00'), + ], + dateRangeEndInclusive: false, + }, + { + dateRange: [ + new Date('2023-01-10 00:00:00'), + new Date('2023-01-10 00:54:00'), + ], + dateRangeEndInclusive: false, + }, + ]); + }); + + it('returns windows aligned to the granularity if the granularity is larger than the window size', () => { + expect( + getGranularityAlignedTimeWindows( + { + dateRange: [ + new Date('2023-01-10 00:00:00'), + new Date('2023-01-10 00:10:00'), + ], + granularity: '1 minute', + timestampValueExpression: 'TimestampTime', + } as ChartConfigWithDateRange & { granularity: string }, + [ + 30, // 30s + 60, // 1m + 5 * 60, // 5m + ], + ), + ).toEqual([ + { + dateRange: [ + new Date('2023-01-10 00:09:00'), // window is expanded beyond the desired 30s, to align to 1m granularity + new Date('2023-01-10 00:10:00'), + ], + dateRangeEndInclusive: undefined, + }, + { + dateRange: [ + new Date('2023-01-10 00:08:00'), // Second window is 1m (as desired) and aligned to granularity + new Date('2023-01-10 00:09:00'), + ], + dateRangeEndInclusive: false, + }, + { + dateRange: [ + new Date('2023-01-10 00:03:00'), // Third window is 5m (as desired) and aligned to granularity + new Date('2023-01-10 00:08:00'), + ], + dateRangeEndInclusive: false, + }, + { + dateRange: [ + new Date('2023-01-10 00:00:00'), // Fourth window is shortened to fit within the overall date range, but still aligned to granularity + new Date('2023-01-10 00:03:00'), + ], + dateRangeEndInclusive: false, + }, + ]); + }); + + it('Skips windows that would be double-queried due to alignment', () => { + expect( + getGranularityAlignedTimeWindows( + { + dateRange: [ + new Date('2023-01-10 00:08:00'), + new Date('2023-01-10 00:10:00'), + ], + granularity: '1 minute', + timestampValueExpression: 'TimestampTime', + } as ChartConfigWithDateRange & { granularity: string }, + [ + 15, // 15s + ], + ), + ).toEqual([ + { + dateRange: [ + new Date('2023-01-10 00:09:00'), // window is expanded beyond the desired 30s, to align to 1m granularity + new Date('2023-01-10 00:10:00'), + ], + dateRangeEndInclusive: undefined, + }, + { + dateRange: [ + new Date('2023-01-10 00:08:00'), + new Date('2023-01-10 00:09:00'), + ], + dateRangeEndInclusive: false, + }, + ]); + }); + + it('returns windows aligned to the granularity if the granularity is smaller than the window size', () => { + expect( + getGranularityAlignedTimeWindows( + { + dateRange: [ + new Date('2023-01-09 22:00:40'), + new Date('2023-01-10 00:00:30'), + ], + granularity: '1 minute', + timestampValueExpression: 'TimestampTime', + dateRangeEndInclusive: true, + } as ChartConfigWithDateRange & { granularity: string }, + [ + 15 * 60, // 15m + 30 * 60, // 30m + ], + ), + ).toEqual([ + { + dateRange: [ + new Date('2023-01-09 23:45:00'), // Window is lengthened to align to granularity + new Date('2023-01-10 00:00:30'), + ], + dateRangeEndInclusive: true, + }, + { + dateRange: [ + new Date('2023-01-09 23:15:00'), + new Date('2023-01-09 23:45:00'), + ], + dateRangeEndInclusive: false, + }, + { + dateRange: [ + new Date('2023-01-09 22:45:00'), + new Date('2023-01-09 23:15:00'), + ], + dateRangeEndInclusive: false, + }, + { + dateRange: [ + new Date('2023-01-09 22:15:00'), + new Date('2023-01-09 22:45:00'), + ], + dateRangeEndInclusive: false, + }, + { + dateRange: [ + new Date('2023-01-09 22:00:40'), // Window is shortened to fit within the overall date range + new Date('2023-01-09 22:15:00'), + ], + dateRangeEndInclusive: false, + }, + ]); + }); + + it('does not return a window that starts before the overall start date', () => { + expect( + getGranularityAlignedTimeWindows( + { + dateRange: [ + new Date('2023-01-10 00:00:30'), + new Date('2023-01-10 00:02:00'), + ], + granularity: '1 minute', + timestampValueExpression: 'TimestampTime', + } as ChartConfigWithDateRange & { granularity: string }, + [ + 60, // 1m + ], + ), + ).toEqual([ + { + dateRange: [ + new Date('2023-01-10 00:01:00'), + new Date('2023-01-10 00:02:00'), + ], + dateRangeEndInclusive: undefined, + }, + { + dateRange: [ + new Date('2023-01-10 00:00:30'), // Window is shortened to fit within the overall date range + new Date('2023-01-10 00:01:00'), + ], + dateRangeEndInclusive: false, + }, + ]); + }); + }); + + describe('useQueriedChartConfig', () => { + let queryClient: QueryClient; + let wrapper: React.ComponentType<{ children: any }>; + let mockClickhouseClient: jest.Mocked; + + beforeEach(() => { + jest.clearAllMocks(); + + queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + }, + }, + }); + + wrapper = ({ children }) => ( + + {children} + + ); + + mockClickhouseClient = { + queryChartConfig: jest.fn(), + } as unknown as jest.Mocked; + + jest.mocked(useClickhouseClient).mockReturnValue(mockClickhouseClient); + }); + + it('fetches data without chunking when no dateRange is provided', async () => { + const config = createMockChartConfig({ + dateRange: undefined, + granularity: '1 minute', + }); + + const mockResponse = createMockQueryResponse([ + { + 'count()': '71', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-01T00:00:00Z', + }, + { + 'count()': '73', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-02T00:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(1); + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledWith({ + config, + metadata: expect.any(Object), + opts: { + abort_signal: expect.any(AbortSignal), + }, + }); + expect(result.current.data).toEqual({ + data: mockResponse.data, + meta: mockResponse.meta, + rows: mockResponse.rows, + isComplete: true, + }); + expect(result.current.isLoading).toBe(false); + expect(result.current.isPending).toBe(false); + }); + + it('fetches data without chunking when no granularity is provided', async () => { + const config = createMockChartConfig({ + dateRange: [new Date('2025-10-01'), new Date('2025-10-02')], + granularity: undefined, + }); + + const mockResponse = createMockQueryResponse([ + { + 'count()': '71', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-01T00:00:00Z', + }, + { + 'count()': '73', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-02T00:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(1); + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledWith({ + config, + metadata: expect.any(Object), + opts: { + abort_signal: expect.any(AbortSignal), + }, + }); + expect(result.current.data).toEqual({ + data: mockResponse.data, + meta: mockResponse.meta, + rows: mockResponse.rows, + isComplete: true, + }); + expect(result.current.isLoading).toBe(false); + expect(result.current.isPending).toBe(false); + }); + + it('fetches data without chunking when no timestampValueExpression is provided', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '1 hour', + timestampValueExpression: undefined, + }); + + const mockResponse = createMockQueryResponse([ + { + 'count()': '71', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-01T00:00:00Z', + }, + { + 'count()': '73', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-02T00:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + // Should only be called once since chunking is disabled without timestampValueExpression + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(1); + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledWith({ + config, + metadata: expect.any(Object), + opts: { + abort_signal: expect.any(AbortSignal), + }, + }); + expect(result.current.data).toEqual({ + data: mockResponse.data, + meta: mockResponse.meta, + rows: mockResponse.rows, + isComplete: true, + }); + }); + + it('fetches data without chunking for metric chart configs', async () => { + const config: ChartConfigWithOptDateRange = { + select: [ + { + aggFn: 'min', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'system.network.io', + metricType: MetricsDataType.Sum, + }, + ], + where: '', + whereLanguage: 'lucene', + granularity: '1 minute', + from: { + databaseName: 'default', + tableName: '', + }, + timestampValueExpression: 'TimeUnix', + dateRange: [ + new Date('2025-10-06T18:35:47.599Z'), + new Date('2025-10-10T19:35:47.599Z'), + ], + connection: 'foo', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: '', + 'exponential histogram': '', + }, + limit: { + limit: 100000, + }, + }; + + const mockResponse = createMockQueryResponse([ + { + 'count()': '71', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-01T00:00:00Z', + }, + { + 'count()': '73', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-02T00:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + // Should only be called once since chunking is disabled without timestampValueExpression + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(1); + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledWith({ + config, + metadata: expect.any(Object), + opts: { + abort_signal: expect.any(AbortSignal), + }, + }); + expect(result.current.data).toEqual({ + data: mockResponse.data, + meta: mockResponse.meta, + rows: mockResponse.rows, + isComplete: true, + }); + }); + + it('fetches data without chunking when disableQueryChunking is true', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '1 hour', + }); + + const mockResponse = createMockQueryResponse([ + { + 'count()': '71', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-01T00:00:00Z', + }, + { + 'count()': '73', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-02T00:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook( + () => useQueriedChartConfig(config, { disableQueryChunking: true }), + { + wrapper, + }, + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + // Should only be called once since chunking is explicitly disabled + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(1); + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledWith({ + config, + metadata: expect.any(Object), + opts: { + abort_signal: expect.any(AbortSignal), + }, + }); + expect(result.current.data).toEqual({ + data: mockResponse.data, + meta: mockResponse.meta, + rows: mockResponse.rows, + isComplete: true, + }); + }); + + it('fetches data with chunking when granularity and date range are provided', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '3 hour', + }); + + const mockResponse1 = createMockQueryResponse([ + { + 'count()': '71', + __hdx_time_bucket: '2025-10-01T18:00:00Z', + }, + { + 'count()': '72', + __hdx_time_bucket: '2025-10-01T19:00:00Z', + }, + ]); + + const mockResponse2 = createMockQueryResponse([ + { + 'count()': '73', + __hdx_time_bucket: '2025-10-01T12:00:00Z', + }, + { + 'count()': '74', + __hdx_time_bucket: '2025-10-01T14:00:00Z', + }, + ]); + + const mockResponse3 = createMockQueryResponse([ + { + 'count()': '75', + __hdx_time_bucket: '2025-10-01T01:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig + .mockResolvedValueOnce(mockResponse1) + .mockResolvedValueOnce(mockResponse2) + .mockResolvedValueOnce(mockResponse3); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(3); + const clickHouseCalls = mockClickhouseClient.queryChartConfig.mock.calls; + expect(clickHouseCalls[0][0].config).toEqual({ + ...config, + dateRange: [ + new Date('2025-10-01T18:00:00.000Z'), + new Date('2025-10-02T00:00:00.000Z'), + ], + dateRangeEndInclusive: undefined, + }); + + expect(clickHouseCalls[1][0].config).toEqual({ + ...config, + dateRange: [ + new Date('2025-10-01T12:00:00.000Z'), + new Date('2025-10-01T18:00:00.000Z'), + ], + dateRangeEndInclusive: false, + }); + + expect(clickHouseCalls[2][0].config).toEqual({ + ...config, + dateRange: [ + new Date('2025-10-01T00:00:00.000Z'), + new Date('2025-10-01T12:00:00.000Z'), + ], + dateRangeEndInclusive: false, + }); + + expect(result.current.data).toEqual({ + data: [ + ...mockResponse3.data, + ...mockResponse2.data, + ...mockResponse1.data, + ], + meta: mockResponse1.meta, + rows: 5, + isComplete: true, + }); + expect(result.current.isLoading).toBe(false); + expect(result.current.isPending).toBe(false); + }); + + it('remains in a fetching state, with partial data until all data is loaded', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '3 hour', + }); + + const mockResponse1 = createMockQueryResponse([ + { + 'count()': '71', + __hdx_time_bucket: '2025-10-01T18:00:00Z', + }, + { + 'count()': '72', + __hdx_time_bucket: '2025-10-01T19:00:00Z', + }, + ]); + + const mockResponse2 = createMockQueryResponse([ + { + 'count()': '73', + __hdx_time_bucket: '2025-10-01T12:00:00Z', + }, + { + 'count()': '74', + __hdx_time_bucket: '2025-10-01T14:00:00Z', + }, + ]); + + // Create a promise that we can control when it resolves + let resolveMockResponse3: (value: ResponseJSON) => void | undefined; + const mockResponse3 = new Promise>(resolve => { + resolveMockResponse3 = resolve; + }); + + mockClickhouseClient.queryChartConfig + .mockResolvedValueOnce(mockResponse1) + .mockResolvedValueOnce(mockResponse2) + .mockResolvedValueOnce(mockResponse3); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isPending).toBe(false)); + + // Partial response is available + expect(result.current.data).toEqual({ + data: [...mockResponse2.data, ...mockResponse1.data], + meta: mockResponse1.meta, + rows: 4, + isComplete: false, + }); + expect(result.current.isFetching).toBe(true); + expect(result.current.isLoading).toBe(false); // isLoading is false because we have partial data + expect(result.current.isSuccess).toBe(true); // isSuccess is true because we have partial data + + // Resolve the final promise to simulate data arriving + const mockResponse3Data = createMockQueryResponse([ + { + 'count()': '75', + __hdx_time_bucket: '2025-10-01T01:00:00Z', + }, + ]); + + resolveMockResponse3!(mockResponse3Data); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + expect(result.current.data).toEqual({ + data: [ + ...mockResponse3Data.data, + ...mockResponse2.data, + ...mockResponse1.data, + ], + meta: mockResponse1.meta, + rows: 5, + isComplete: true, + }); + }); + + it('is in a loading state until the first chunk has loaded', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '3 hour', + }); + + // Create a promise that we can control when it resolves + let resolveMockResponse1: (value: ResponseJSON) => void | undefined; + const mockResponse1Promise = new Promise>(resolve => { + resolveMockResponse1 = resolve; + }); + + mockClickhouseClient.queryChartConfig.mockResolvedValueOnce( + mockResponse1Promise, + ); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + // Should be in loading state before first chunk + expect(result.current.isLoading).toBe(true); + expect(result.current.isPending).toBe(true); + expect(result.current.data).toBeUndefined(); + + // Resolve the first chunk + const mockResponse1 = createMockQueryResponse([ + { + 'count()': '71', + __hdx_time_bucket: '2025-10-01T18:00:00Z', + }, + ]); + resolveMockResponse1!(mockResponse1); + + await waitFor(() => expect(result.current.isLoading).toBe(false)); + await waitFor(() => expect(result.current.isPending).toBe(false)); + + // Should now have data from first chunk + expect(result.current.data).toEqual({ + data: mockResponse1.data, + meta: mockResponse1.meta, + rows: 1, + isComplete: false, + }); + }); + + it('calls onError callback if provided when a query error occurs', async () => { + const mockError = new Error('Query failed'); + mockClickhouseClient.queryChartConfig.mockRejectedValue(mockError); + + const onError = jest.fn(); + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '3 hour', + }); + + const { result } = renderHook( + () => useQueriedChartConfig(config, { onError, retry: false }), + { + wrapper, + }, + ); + + await waitFor(() => expect(result.current.isError).toBe(true)); + + expect(onError).toHaveBeenCalledWith(mockError); + expect(result.current.error).toBe(mockError); + }); + + it('does not make requests if it is disabled', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '3 hour', + }); + + const mockResponse = createMockQueryResponse([ + { + 'count()': '71', + __hdx_time_bucket: '2025-10-01T18:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook( + () => useQueriedChartConfig(config, { enabled: false }), + { + wrapper, + }, + ); + + // Wait a bit to ensure no calls are made + await new Promise(resolve => setTimeout(resolve, 100)); + + expect(mockClickhouseClient.queryChartConfig).not.toHaveBeenCalled(); + expect(result.current.isPending).toBe(true); + expect(result.current.data).toBeUndefined(); + }); + + it('uses different query keys for the same config when one sets disableQueryChunking', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '3 hour', + }); + + const mockResponseChunked = createMockQueryResponse([ + { + 'count()': '50', + __hdx_time_bucket: '2025-10-01T18:00:00Z', + }, + ]); + + const mockResponseNonChunked = createMockQueryResponse([ + { + 'count()': '100', + __hdx_time_bucket: '2025-10-01T12:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue( + mockResponseChunked, + ); + + const { result: result1 } = renderHook( + () => useQueriedChartConfig(config), + { + wrapper, + }, + ); + + await waitFor(() => expect(result1.current.isSuccess).toBe(true)); + await waitFor(() => expect(result1.current.isFetching).toBe(false)); + + // Should have been called multiple times for chunked query + const chunkedCallCount = + mockClickhouseClient.queryChartConfig.mock.calls.length; + expect(chunkedCallCount).toBeGreaterThan(1); + expect(result1.current.data?.rows).toBeGreaterThan(1); + + // Second render with same config but disableQueryChunking=true + mockClickhouseClient.queryChartConfig.mockResolvedValue( + mockResponseNonChunked, + ); + + const { result: result2 } = renderHook( + () => useQueriedChartConfig(config, { disableQueryChunking: true }), + { + wrapper, + }, + ); + + await waitFor(() => expect(result2.current.isSuccess).toBe(true)); + await waitFor(() => expect(result2.current.isFetching).toBe(false)); + + // Should have made a new request (not using cached chunked data) + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes( + chunkedCallCount + 1, + ); + expect(result2.current.data?.rows).toBe(1); + + // The original query should still have its chunked data + expect(result1.current.data?.rows).toBeGreaterThan(1); + }); + }); +}); diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index e8bae398f..8115e005a 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -1,58 +1,221 @@ -import { useEffect } from 'react'; -import objectHash from 'object-hash'; import { - ChSql, chSqlToAliasMap, ClickHouseQueryError, - inferNumericColumn, - inferTimestampColumn, parameterizedQueryToSql, ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; -import { renderChartConfig } from '@hyperdx/common-utils/dist/renderChartConfig'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import { + DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS, + isMetricChartConfig, + isUsingGranularity, + renderChartConfig, +} from '@hyperdx/common-utils/dist/renderChartConfig'; import { format } from '@hyperdx/common-utils/dist/sqlFormatter'; -import { ChartConfigWithOptDateRange } from '@hyperdx/common-utils/dist/types'; -import { useQuery, UseQueryOptions } from '@tanstack/react-query'; +import { + ChartConfigWithDateRange, + ChartConfigWithOptDateRange, +} from '@hyperdx/common-utils/dist/types'; +import { + experimental_streamedQuery as streamedQuery, + useQuery, + UseQueryOptions, +} from '@tanstack/react-query'; +import { + convertDateRangeToGranularityString, + toStartOfInterval, +} from '@/ChartUtils'; import { useClickhouseClient } from '@/clickhouse'; import { IS_MTVIEWS_ENABLED } from '@/config'; import { buildMTViewSelectQuery } from '@/hdxMTViews'; import { getMetadata } from '@/metadata'; +import { generateTimeWindowsDescending } from '@/utils/searchWindows'; interface AdditionalUseQueriedChartConfigOptions { onError?: (error: Error | ClickHouseQueryError) => void; + /** + * By default, queries with large date ranges are split into multiple smaller queries to + * avoid overloading the ClickHouse server and running into timeouts. In some cases, such + * as when data is being sampled across the entire range, this chunking is not desirable + * and can be disabled. + */ + disableQueryChunking?: boolean; +} + +type TimeWindow = { + dateRange: [Date, Date]; + dateRangeEndInclusive?: boolean; +}; + +type TQueryFnData = Pick, 'data' | 'meta' | 'rows'> & { + isComplete: boolean; +}; + +const shouldUseChunking = ( + config: ChartConfigWithOptDateRange, +): config is ChartConfigWithDateRange & { + granularity: string; +} => { + // Granularity is required for chunking, otherwise we could break other group-bys. + if (!isUsingGranularity(config)) return false; + + // Date range is required for chunking, otherwise we'd have infinite chunks, or some unbounded chunk(s). + if (!config.dateRange) return false; + + // TODO: enable chunking for metric charts when we're confident chunking will not break + // complex metric queries. + if (isMetricChartConfig(config)) return false; + + return true; +}; + +export const getGranularityAlignedTimeWindows = ( + config: ChartConfigWithDateRange & { granularity: string }, + windowDurationsSeconds?: number[], +): TimeWindow[] => { + const [startDate, endDate] = config.dateRange; + const windowsUnaligned = generateTimeWindowsDescending( + startDate, + endDate, + windowDurationsSeconds, + ); + + const granularity = + config.granularity === 'auto' + ? convertDateRangeToGranularityString( + config.dateRange, + DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS, + ) + : config.granularity; + + const windows = []; + for (const [index, window] of windowsUnaligned.entries()) { + // Align windows to chart buckets + const alignedStart = + index === windowsUnaligned.length - 1 + ? window.startTime + : toStartOfInterval(window.startTime, granularity); + const alignedEnd = + index === 0 ? endDate : toStartOfInterval(window.endTime, granularity); + + // Skip windows that are covered by the previous window after it was aligned + if ( + !windows.length || + alignedStart < windows[windows.length - 1].dateRange[0] + ) { + windows.push({ + dateRange: [alignedStart, alignedEnd] as [Date, Date], + // Ensure that windows don't overlap by making all but the first (most recent) exclusive + dateRangeEndInclusive: + index === 0 ? config.dateRangeEndInclusive : false, + }); + } + } + + return windows; +}; + +async function* fetchDataInChunks( + config: ChartConfigWithOptDateRange, + clickhouseClient: ClickhouseClient, + signal: AbortSignal, + disableQueryChunking: boolean = false, +) { + const windows = + !disableQueryChunking && shouldUseChunking(config) + ? getGranularityAlignedTimeWindows(config) + : ([undefined] as const); + + if (IS_MTVIEWS_ENABLED) { + const { dataTableDDL, mtViewDDL, renderMTViewConfig } = + await buildMTViewSelectQuery(config); + // TODO: show the DDLs in the UI so users can run commands manually + // eslint-disable-next-line no-console + console.log('dataTableDDL:', dataTableDDL); + // eslint-disable-next-line no-console + console.log('mtViewDDL:', mtViewDDL); + await renderMTViewConfig(); + } + + for (let i = 0; i < windows.length; i++) { + const window = windows[i]; + + const windowedConfig = { + ...config, + ...(window ?? {}), + }; + + const result = await clickhouseClient.queryChartConfig({ + config: windowedConfig, + metadata: getMetadata(), + opts: { + abort_signal: signal, + }, + }); + + yield { chunk: result, isComplete: i === windows.length - 1 }; + } } -// used for charting +/** + * A hook providing data queried based on the provided chart config. + * + * If all of the following are true, the query will be chunked into multiple smaller queries: + * - The config includes a dateRange, granularity, and timestampValueExpression + * - `options.disableQueryChunking` is falsy + * + * For chunked queries, note the following: + * - `config.limit`, if provided, is applied to each chunk, so the total number + * of rows returned may be up to `limit * number_of_chunks`. + * - The returned data will be ordered within each chunk, and chunks will + * be ordered oldest-first, by the `timestampValueExpression`. + * - `isPending` is true until the first chunk is fetched. Once the first chunk + * is available, `isPending` will be false and `isSuccess` will be true. + * `isFetching` will be true until all chunks have been fetched. + * - `data.isComplete` indicates whether all chunks have been fetched. + */ export function useQueriedChartConfig( config: ChartConfigWithOptDateRange, - options?: Partial>> & + options?: Partial> & AdditionalUseQueriedChartConfigOptions, ) { const clickhouseClient = useClickhouseClient(); - const query = useQuery, ClickHouseQueryError | Error>({ - queryKey: [config], - queryFn: async ({ signal }) => { - let query = null; - if (IS_MTVIEWS_ENABLED) { - const { dataTableDDL, mtViewDDL, renderMTViewConfig } = - await buildMTViewSelectQuery(config); - // TODO: show the DDLs in the UI so users can run commands manually - // eslint-disable-next-line no-console - console.log('dataTableDDL:', dataTableDDL); - // eslint-disable-next-line no-console - console.log('mtViewDDL:', mtViewDDL); - query = await renderMTViewConfig(); - } - return clickhouseClient.queryChartConfig({ - config, - metadata: getMetadata(), - opts: { - abort_signal: signal, - }, - }); - }, + const query = useQuery({ + // Include disableQueryChunking in the query key to ensure that queries with the + // same config but different disableQueryChunking values do not share a query + queryKey: [config, options?.disableQueryChunking ?? false], + queryFn: streamedQuery({ + streamFn: context => + fetchDataInChunks( + config, + clickhouseClient, + context.signal, + options?.disableQueryChunking, + ), + /** + * This mode ensures that data remains in the cache until the next full streamed result is available. + * By default, the cache would be cleared before new data starts arriving, which results in the query briefly + * going back into the loading/pending state when multiple observers are sharing the query result resulting + * in flickering or render loops. + */ + refetchMode: 'replace', + initialValue: { + data: [], + meta: [], + rows: 0, + isComplete: false, + } as TQueryFnData, + reducer: (acc, { chunk, isComplete }) => { + return { + data: [...(chunk.data || []), ...(acc?.data || [])], + meta: chunk.meta, + rows: (acc?.rows || 0) + (chunk.rows || 0), + isComplete, + }; + }, + }), retry: 1, refetchOnWindowFocus: false, ...options, diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 16e5077d2..db5a2696d 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -23,6 +23,11 @@ import api from '@/api'; import { getClickhouseClient } from '@/clickhouse'; import { getMetadata } from '@/metadata'; import { omit } from '@/utils'; +import { + generateTimeWindowsAscending, + generateTimeWindowsDescending, + TimeWindow, +} from '@/utils/searchWindows'; type TQueryKey = readonly [ string, @@ -37,21 +42,6 @@ function queryKeyFn( return [prefix, config, queryTimeout]; } -// Time window configuration - progressive bucketing strategy -const TIME_WINDOWS_MS = [ - 6 * 60 * 60 * 1000, // 6h - 6 * 60 * 60 * 1000, // 6h - 12 * 60 * 60 * 1000, // 12h - 24 * 60 * 60 * 1000, // 24h -]; - -type TimeWindow = { - startTime: Date; - endTime: Date; - windowIndex: number; - direction: 'ASC' | 'DESC'; -}; - type TPageParam = { windowIndex: number; offset: number; @@ -69,65 +59,6 @@ type TData = { pageParams: TPageParam[]; }; -// Generate time windows from date range using progressive bucketing, starting at the end of the date range -function generateTimeWindowsDescending( - startDate: Date, - endDate: Date, -): TimeWindow[] { - const windows: TimeWindow[] = []; - let currentEnd = new Date(endDate); - let windowIndex = 0; - - while (currentEnd > startDate) { - const windowSize = - TIME_WINDOWS_MS[windowIndex] || - TIME_WINDOWS_MS[TIME_WINDOWS_MS.length - 1]; // use largest window size - const windowStart = new Date( - Math.max(currentEnd.getTime() - windowSize, startDate.getTime()), - ); - - windows.push({ - endTime: new Date(currentEnd), - startTime: windowStart, - windowIndex, - direction: 'DESC', - }); - - currentEnd = windowStart; - windowIndex++; - } - - return windows; -} - -// Generate time windows from date range using progressive bucketing, starting at the beginning of the date range -function generateTimeWindowsAscending(startDate: Date, endDate: Date) { - const windows: TimeWindow[] = []; - let currentStart = new Date(startDate); - let windowIndex = 0; - - while (currentStart < endDate) { - const windowSize = - TIME_WINDOWS_MS[windowIndex] || - TIME_WINDOWS_MS[TIME_WINDOWS_MS.length - 1]; // use largest window size - const windowEnd = new Date( - Math.min(currentStart.getTime() + windowSize, endDate.getTime()), - ); - - windows.push({ - startTime: new Date(currentStart), - endTime: windowEnd, - windowIndex, - direction: 'ASC', - }); - - currentStart = windowEnd; - windowIndex++; - } - - return windows; -} - // Get time window from page param function getTimeWindowFromPageParam( config: ChartConfigWithOptTimestamp, diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index d229d1b52..43f956605 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -141,10 +141,15 @@ function usePatterns({ limit: { limit: samples }, }); - const { data: sampleRows } = useQueriedChartConfig( - configWithPrimaryAndPartitionKey ?? config, // `config` satisfying type, never used due to `enabled` check - { enabled: configWithPrimaryAndPartitionKey != null && enabled }, - ); + const { data: sampleRows, isLoading: isSampleLoading } = + useQueriedChartConfig( + configWithPrimaryAndPartitionKey ?? config, // `config` satisfying type, never used due to `enabled` check + { + enabled: configWithPrimaryAndPartitionKey != null && enabled, + // Disable chunking to ensure we get the desired sample size + disableQueryChunking: true, + }, + ); const { data: pyodide, isLoading: isLoadingPyodide } = usePyodide({ enabled, @@ -191,7 +196,7 @@ function usePatterns({ return { ...query, - isLoading: query.isLoading || isLoadingPyodide, + isLoading: query.isLoading || isSampleLoading || isLoadingPyodide, patternQueryConfig: configWithPrimaryAndPartitionKey, }; } diff --git a/packages/app/src/utils/searchWindows.ts b/packages/app/src/utils/searchWindows.ts new file mode 100644 index 000000000..7cbac8629 --- /dev/null +++ b/packages/app/src/utils/searchWindows.ts @@ -0,0 +1,79 @@ +export const DEFAULT_TIME_WINDOWS_SECONDS = [ + 6 * 60 * 60, // 6h + 6 * 60 * 60, // 6h + 12 * 60 * 60, // 12h + 24 * 60 * 60, // 24h +]; + +export type TimeWindow = { + startTime: Date; + endTime: Date; + windowIndex: number; + direction: 'ASC' | 'DESC'; +}; + +// Generate time windows from date range using progressive bucketing, starting at the end of the date range +export function generateTimeWindowsDescending( + startDate: Date, + endDate: Date, + windowDurationsSeconds: number[] = DEFAULT_TIME_WINDOWS_SECONDS, +): TimeWindow[] { + const windows: TimeWindow[] = []; + let currentEnd = new Date(endDate); + let windowIndex = 0; + + while (currentEnd > startDate) { + const windowSizeSeconds = + windowDurationsSeconds[windowIndex] || + windowDurationsSeconds[windowDurationsSeconds.length - 1]; // use largest window size + const windowSizeMs = windowSizeSeconds * 1000; + const windowStart = new Date( + Math.max(currentEnd.getTime() - windowSizeMs, startDate.getTime()), + ); + + windows.push({ + endTime: new Date(currentEnd), + startTime: windowStart, + windowIndex, + direction: 'DESC', + }); + + currentEnd = windowStart; + windowIndex++; + } + + return windows; +} + +// Generate time windows from date range using progressive bucketing, starting at the beginning of the date range +export function generateTimeWindowsAscending( + startDate: Date, + endDate: Date, + windowDurationsSeconds: number[] = DEFAULT_TIME_WINDOWS_SECONDS, +) { + const windows: TimeWindow[] = []; + let currentStart = new Date(startDate); + let windowIndex = 0; + + while (currentStart < endDate) { + const windowSizeSeconds = + windowDurationsSeconds[windowIndex] || + windowDurationsSeconds[windowDurationsSeconds.length - 1]; // use largest window size + const windowSizeMs = windowSizeSeconds * 1000; + const windowEnd = new Date( + Math.min(currentStart.getTime() + windowSizeMs, endDate.getTime()), + ); + + windows.push({ + startTime: new Date(currentStart), + endTime: windowEnd, + windowIndex, + direction: 'ASC', + }); + + currentStart = windowEnd; + windowIndex++; + } + + return windows; +} diff --git a/packages/common-utils/src/renderChartConfig.ts b/packages/common-utils/src/renderChartConfig.ts index 0c89096d8..59c57a678 100644 --- a/packages/common-utils/src/renderChartConfig.ts +++ b/packages/common-utils/src/renderChartConfig.ts @@ -45,6 +45,9 @@ import { splitAndTrimWithBracket, } from '@/utils'; +/** The default maximum number of buckets setting when determining a bucket duration for 'auto' granularity */ +export const DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS = 60; + // FIXME: SQLParser.ColumnRef is incomplete type ColumnRef = SQLParser.ColumnRef & { array_index?: { @@ -71,7 +74,7 @@ export function isUsingGroupBy( return chartConfig.groupBy != null && chartConfig.groupBy.length > 0; } -function isUsingGranularity( +export function isUsingGranularity( chartConfig: ChartConfigWithOptDateRange, ): chartConfig is Omit< Omit, 'dateRange'>, @@ -467,7 +470,10 @@ function timeBucketExpr({ const unsafeInterval = { UNSAFE_RAW_SQL: interval === 'auto' && Array.isArray(dateRange) - ? convertDateRangeToGranularityString(dateRange, 60) + ? convertDateRangeToGranularityString( + dateRange, + DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS, + ) : interval, }; @@ -929,7 +935,10 @@ function renderDeltaExpression( ) { const interval = chartConfig.granularity === 'auto' && Array.isArray(chartConfig.dateRange) - ? convertDateRangeToGranularityString(chartConfig.dateRange, 60) + ? convertDateRangeToGranularityString( + chartConfig.dateRange, + DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS, + ) : chartConfig.granularity; const intervalInSeconds = convertGranularityToSeconds(interval ?? ''); @@ -1076,7 +1085,10 @@ async function translateMetricChartConfig( includedDataInterval: chartConfig.granularity === 'auto' && Array.isArray(chartConfig.dateRange) - ? convertDateRangeToGranularityString(chartConfig.dateRange, 60) + ? convertDateRangeToGranularityString( + chartConfig.dateRange, + DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS, + ) : chartConfig.granularity, }, metadata, @@ -1190,7 +1202,10 @@ async function translateMetricChartConfig( includedDataInterval: chartConfig.granularity === 'auto' && Array.isArray(chartConfig.dateRange) - ? convertDateRangeToGranularityString(chartConfig.dateRange, 60) + ? convertDateRangeToGranularityString( + chartConfig.dateRange, + DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS, + ) : chartConfig.granularity, } as ChartConfigWithOptDateRangeEx; diff --git a/yarn.lock b/yarn.lock index 841b918c2..e08dea1d7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4585,7 +4585,7 @@ __metadata: "@storybook/react": "npm:^8.1.5" "@storybook/test": "npm:^8.1.5" "@tabler/icons-react": "npm:^3.5.0" - "@tanstack/react-query": "npm:^5.56.2" + "@tanstack/react-query": "npm:^5.90.2" "@tanstack/react-query-devtools": "npm:^5.56.2" "@tanstack/react-table": "npm:^8.7.9" "@tanstack/react-virtual": "npm:^3.0.1" @@ -9392,10 +9392,10 @@ __metadata: languageName: node linkType: hard -"@tanstack/query-core@npm:5.56.2": - version: 5.56.2 - resolution: "@tanstack/query-core@npm:5.56.2" - checksum: 10c0/54ff55f02b01f6ba089f4965bfd46f430c18ce7e11d874de04c4d58cc8f698598b41e1c017ba029d08ae75e321e546b26f1ea7f788474db265eeba46e780f2f6 +"@tanstack/query-core@npm:5.90.2": + version: 5.90.2 + resolution: "@tanstack/query-core@npm:5.90.2" + checksum: 10c0/695a7450b0bb9f6dd21bebeacfc962dfc886631a3b3a13c33a842ef719b4c3dd30c15febe8c1ade6902a85e0f387c51a97570f430cc8f5c7032ff737d6410597 languageName: node linkType: hard @@ -9418,14 +9418,14 @@ __metadata: languageName: node linkType: hard -"@tanstack/react-query@npm:^5.56.2": - version: 5.56.2 - resolution: "@tanstack/react-query@npm:5.56.2" +"@tanstack/react-query@npm:^5.90.2": + version: 5.90.2 + resolution: "@tanstack/react-query@npm:5.90.2" dependencies: - "@tanstack/query-core": "npm:5.56.2" + "@tanstack/query-core": "npm:5.90.2" peerDependencies: react: ^18 || ^19 - checksum: 10c0/6e883b4ca1948f990215b7bce194251faf13a79c6ecf3f3c660af6c6788ed113ab629cefdafb496dfb04866f12dd48d7314e936b75c881b6749127b6496ac8fd + checksum: 10c0/22e76626a59890409858521b0e42b49219126a4ea5ed79eaa48a267959175dfdd28b30b9b03a415dccf703d95c18100a9d8917679818f6d2adc26d6c5f96a4d6 languageName: node linkType: hard From cd794bc738d88496b57095aa118eb5a2a2bf2ce9 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 15 Oct 2025 11:36:32 -0400 Subject: [PATCH 2/6] chore: Upgrade react-query-devtools --- packages/app/package.json | 2 +- yarn.lock | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index 4962530f5..80ea17fe4 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -42,7 +42,7 @@ "@microsoft/fetch-event-source": "^2.0.1", "@tabler/icons-react": "^3.5.0", "@tanstack/react-query": "^5.90.2", - "@tanstack/react-query-devtools": "^5.56.2", + "@tanstack/react-query-devtools": "^5.90.2", "@tanstack/react-table": "^8.7.9", "@tanstack/react-virtual": "^3.0.1", "@uiw/codemirror-theme-atomone": "^4.23.3", diff --git a/yarn.lock b/yarn.lock index e08dea1d7..f82b165d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4586,7 +4586,7 @@ __metadata: "@storybook/test": "npm:^8.1.5" "@tabler/icons-react": "npm:^3.5.0" "@tanstack/react-query": "npm:^5.90.2" - "@tanstack/react-query-devtools": "npm:^5.56.2" + "@tanstack/react-query-devtools": "npm:^5.90.2" "@tanstack/react-table": "npm:^8.7.9" "@tanstack/react-virtual": "npm:^3.0.1" "@testing-library/jest-dom": "npm:^6.4.2" @@ -9399,22 +9399,22 @@ __metadata: languageName: node linkType: hard -"@tanstack/query-devtools@npm:5.56.1": - version: 5.56.1 - resolution: "@tanstack/query-devtools@npm:5.56.1" - checksum: 10c0/4f50fccf9e731e0fee4b7d2344cd8bf73a9c45816d7361f54ca4ec2df0a0455bf400fa3b578279edc8a4d1013c52c0d90cd50dbffb133edcf52e55868468d6c3 +"@tanstack/query-devtools@npm:5.90.1": + version: 5.90.1 + resolution: "@tanstack/query-devtools@npm:5.90.1" + checksum: 10c0/3b69e5441438acf0e753adbf187abf54b5b2e19d7c6d1e465d97278cb8c248bb86d3be193092d50414e4093cbf014093103517cb523daae003e53c867f3c11c2 languageName: node linkType: hard -"@tanstack/react-query-devtools@npm:^5.56.2": - version: 5.56.2 - resolution: "@tanstack/react-query-devtools@npm:5.56.2" +"@tanstack/react-query-devtools@npm:^5.90.2": + version: 5.90.2 + resolution: "@tanstack/react-query-devtools@npm:5.90.2" dependencies: - "@tanstack/query-devtools": "npm:5.56.1" + "@tanstack/query-devtools": "npm:5.90.1" peerDependencies: - "@tanstack/react-query": ^5.56.2 + "@tanstack/react-query": ^5.90.2 react: ^18 || ^19 - checksum: 10c0/2edebb04800585da1ca479f9289a9265cfcc26c5d8fb1db08e6fbe9fdf370a55a2007416b52f819c2faeab7b919cd51a4afca7731b50b6f3dd164057e4966741 + checksum: 10c0/526d529bf995426ace7511f51a425ce92dfc1b6dd74c9956a3cd7d68950119e97291bced2ff17173bcdb329eae36c68abc211a4dec32d6e92ab537b41c0533c2 languageName: node linkType: hard From 98ed051adc7623b539f2d313412bf2725e683ddf Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 15 Oct 2025 14:27:06 -0400 Subject: [PATCH 3/6] feat: Remove experimental streamedQuery --- packages/app/package.json | 4 +- packages/app/src/hooks/useChartConfig.tsx | 94 +++++++++++++++-------- yarn.lock | 42 +++++----- 3 files changed, 86 insertions(+), 54 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index 80ea17fe4..cd2086661 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -41,8 +41,8 @@ "@mantine/spotlight": "7.9.2", "@microsoft/fetch-event-source": "^2.0.1", "@tabler/icons-react": "^3.5.0", - "@tanstack/react-query": "^5.90.2", - "@tanstack/react-query-devtools": "^5.90.2", + "@tanstack/react-query": "^5.56.2", + "@tanstack/react-query-devtools": "^5.56.2", "@tanstack/react-table": "^8.7.9", "@tanstack/react-virtual": "^3.0.1", "@uiw/codemirror-theme-atomone": "^4.23.3", diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index 8115e005a..de97d10af 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -16,11 +16,7 @@ import { ChartConfigWithDateRange, ChartConfigWithOptDateRange, } from '@hyperdx/common-utils/dist/types'; -import { - experimental_streamedQuery as streamedQuery, - useQuery, - UseQueryOptions, -} from '@tanstack/react-query'; +import { useQuery, UseQueryOptions } from '@tanstack/react-query'; import { convertDateRangeToGranularityString, @@ -52,6 +48,11 @@ type TQueryFnData = Pick, 'data' | 'meta' | 'rows'> & { isComplete: boolean; }; +type TChunk = { + chunk: ResponseJSON>; + isComplete: boolean; +}; + const shouldUseChunking = ( config: ChartConfigWithOptDateRange, ): config is ChartConfigWithDateRange & { @@ -158,6 +159,19 @@ async function* fetchDataInChunks( } } +/** Append the given chunk to the given accumulated result */ +function appendChunk( + accumulated: TQueryFnData, + { chunk, isComplete }: TChunk, +): TQueryFnData { + return { + data: [...(chunk.data || []), ...(accumulated?.data || [])], + meta: chunk.meta, + rows: (accumulated?.rows || 0) + (chunk.rows || 0), + isComplete, + }; +} + /** * A hook providing data queried based on the provided chart config. * @@ -186,36 +200,54 @@ export function useQueriedChartConfig( // Include disableQueryChunking in the query key to ensure that queries with the // same config but different disableQueryChunking values do not share a query queryKey: [config, options?.disableQueryChunking ?? false], - queryFn: streamedQuery({ - streamFn: context => - fetchDataInChunks( - config, - clickhouseClient, - context.signal, - options?.disableQueryChunking, - ), - /** - * This mode ensures that data remains in the cache until the next full streamed result is available. - * By default, the cache would be cleared before new data starts arriving, which results in the query briefly - * going back into the loading/pending state when multiple observers are sharing the query result resulting - * in flickering or render loops. - */ - refetchMode: 'replace', - initialValue: { + // TODO: Replace this with `streamedQuery` when it is no longer experimental. Use 'replace' refetch mode. + // https://tanstack.com/query/latest/docs/reference/streamedQuery + queryFn: async context => { + const query = context.client + .getQueryCache() + .find({ queryKey: context.queryKey, exact: true }); + const isRefetch = !!query && query.state.data !== undefined; + + const emptyValue: TQueryFnData = { data: [], meta: [], rows: 0, isComplete: false, - } as TQueryFnData, - reducer: (acc, { chunk, isComplete }) => { - return { - data: [...(chunk.data || []), ...(acc?.data || [])], - meta: chunk.meta, - rows: (acc?.rows || 0) + (chunk.rows || 0), - isComplete, - }; - }, - }), + }; + + const chunks = fetchDataInChunks( + config, + clickhouseClient, + context.signal, + options?.disableQueryChunking, + ); + + let accumulatedChunks: TQueryFnData = emptyValue; + for await (const chunk of chunks) { + if (context.signal.aborted) { + break; + } + + accumulatedChunks = appendChunk(accumulatedChunks, chunk); + + // When refetching, the cache is not updated until all chunks are fetched. + if (!isRefetch) { + context.client.setQueryData( + context.queryKey, + accumulatedChunks, + ); + } + } + + if (isRefetch && !context.signal.aborted) { + context.client.setQueryData( + context.queryKey, + accumulatedChunks, + ); + } + + return context.client.getQueryData(context.queryKey)!; + }, retry: 1, refetchOnWindowFocus: false, ...options, diff --git a/yarn.lock b/yarn.lock index f82b165d4..841b918c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4585,8 +4585,8 @@ __metadata: "@storybook/react": "npm:^8.1.5" "@storybook/test": "npm:^8.1.5" "@tabler/icons-react": "npm:^3.5.0" - "@tanstack/react-query": "npm:^5.90.2" - "@tanstack/react-query-devtools": "npm:^5.90.2" + "@tanstack/react-query": "npm:^5.56.2" + "@tanstack/react-query-devtools": "npm:^5.56.2" "@tanstack/react-table": "npm:^8.7.9" "@tanstack/react-virtual": "npm:^3.0.1" "@testing-library/jest-dom": "npm:^6.4.2" @@ -9392,40 +9392,40 @@ __metadata: languageName: node linkType: hard -"@tanstack/query-core@npm:5.90.2": - version: 5.90.2 - resolution: "@tanstack/query-core@npm:5.90.2" - checksum: 10c0/695a7450b0bb9f6dd21bebeacfc962dfc886631a3b3a13c33a842ef719b4c3dd30c15febe8c1ade6902a85e0f387c51a97570f430cc8f5c7032ff737d6410597 +"@tanstack/query-core@npm:5.56.2": + version: 5.56.2 + resolution: "@tanstack/query-core@npm:5.56.2" + checksum: 10c0/54ff55f02b01f6ba089f4965bfd46f430c18ce7e11d874de04c4d58cc8f698598b41e1c017ba029d08ae75e321e546b26f1ea7f788474db265eeba46e780f2f6 languageName: node linkType: hard -"@tanstack/query-devtools@npm:5.90.1": - version: 5.90.1 - resolution: "@tanstack/query-devtools@npm:5.90.1" - checksum: 10c0/3b69e5441438acf0e753adbf187abf54b5b2e19d7c6d1e465d97278cb8c248bb86d3be193092d50414e4093cbf014093103517cb523daae003e53c867f3c11c2 +"@tanstack/query-devtools@npm:5.56.1": + version: 5.56.1 + resolution: "@tanstack/query-devtools@npm:5.56.1" + checksum: 10c0/4f50fccf9e731e0fee4b7d2344cd8bf73a9c45816d7361f54ca4ec2df0a0455bf400fa3b578279edc8a4d1013c52c0d90cd50dbffb133edcf52e55868468d6c3 languageName: node linkType: hard -"@tanstack/react-query-devtools@npm:^5.90.2": - version: 5.90.2 - resolution: "@tanstack/react-query-devtools@npm:5.90.2" +"@tanstack/react-query-devtools@npm:^5.56.2": + version: 5.56.2 + resolution: "@tanstack/react-query-devtools@npm:5.56.2" dependencies: - "@tanstack/query-devtools": "npm:5.90.1" + "@tanstack/query-devtools": "npm:5.56.1" peerDependencies: - "@tanstack/react-query": ^5.90.2 + "@tanstack/react-query": ^5.56.2 react: ^18 || ^19 - checksum: 10c0/526d529bf995426ace7511f51a425ce92dfc1b6dd74c9956a3cd7d68950119e97291bced2ff17173bcdb329eae36c68abc211a4dec32d6e92ab537b41c0533c2 + checksum: 10c0/2edebb04800585da1ca479f9289a9265cfcc26c5d8fb1db08e6fbe9fdf370a55a2007416b52f819c2faeab7b919cd51a4afca7731b50b6f3dd164057e4966741 languageName: node linkType: hard -"@tanstack/react-query@npm:^5.90.2": - version: 5.90.2 - resolution: "@tanstack/react-query@npm:5.90.2" +"@tanstack/react-query@npm:^5.56.2": + version: 5.56.2 + resolution: "@tanstack/react-query@npm:5.56.2" dependencies: - "@tanstack/query-core": "npm:5.90.2" + "@tanstack/query-core": "npm:5.56.2" peerDependencies: react: ^18 || ^19 - checksum: 10c0/22e76626a59890409858521b0e42b49219126a4ea5ed79eaa48a267959175dfdd28b30b9b03a415dccf703d95c18100a9d8917679818f6d2adc26d6c5f96a4d6 + checksum: 10c0/6e883b4ca1948f990215b7bce194251faf13a79c6ecf3f3c660af6c6788ed113ab629cefdafb496dfb04866f12dd48d7314e936b75c881b6749127b6496ac8fd languageName: node linkType: hard From ade53811d77e4316199f0fcc5c5510c2f70b74ed Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 15 Oct 2025 15:04:08 -0400 Subject: [PATCH 4/6] fix: Fix lint error --- packages/app/src/hooks/useChartConfig.tsx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index de97d10af..689874685 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -16,7 +16,11 @@ import { ChartConfigWithDateRange, ChartConfigWithOptDateRange, } from '@hyperdx/common-utils/dist/types'; -import { useQuery, UseQueryOptions } from '@tanstack/react-query'; +import { + useQuery, + useQueryClient, + UseQueryOptions, +} from '@tanstack/react-query'; import { convertDateRangeToGranularityString, @@ -195,6 +199,7 @@ export function useQueriedChartConfig( AdditionalUseQueriedChartConfigOptions, ) { const clickhouseClient = useClickhouseClient(); + const queryClient = useQueryClient(); const query = useQuery({ // Include disableQueryChunking in the query key to ensure that queries with the @@ -203,7 +208,7 @@ export function useQueriedChartConfig( // TODO: Replace this with `streamedQuery` when it is no longer experimental. Use 'replace' refetch mode. // https://tanstack.com/query/latest/docs/reference/streamedQuery queryFn: async context => { - const query = context.client + const query = queryClient .getQueryCache() .find({ queryKey: context.queryKey, exact: true }); const isRefetch = !!query && query.state.data !== undefined; @@ -232,7 +237,7 @@ export function useQueriedChartConfig( // When refetching, the cache is not updated until all chunks are fetched. if (!isRefetch) { - context.client.setQueryData( + queryClient.setQueryData( context.queryKey, accumulatedChunks, ); @@ -240,13 +245,13 @@ export function useQueriedChartConfig( } if (isRefetch && !context.signal.aborted) { - context.client.setQueryData( + queryClient.setQueryData( context.queryKey, accumulatedChunks, ); } - return context.client.getQueryData(context.queryKey)!; + return queryClient.getQueryData(context.queryKey)!; }, retry: 1, refetchOnWindowFocus: false, From 6ee53de5e3e4e339601396dd4e0d16215f6662b5 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Thu, 16 Oct 2025 09:35:21 -0400 Subject: [PATCH 5/6] fix: Handle date ranges with same start and end date --- .../hooks/__tests__/useChartConfig.test.tsx | 57 +++++++++++++++++++ .../useOffsetPaginatedQuery.test.tsx | 34 +++++++++++ packages/app/src/utils/searchWindows.ts | 24 +++++++- 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx index e4cead466..cc66edc75 100644 --- a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx +++ b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx @@ -286,6 +286,32 @@ describe('useChartConfig', () => { }, ]); }); + + it('returns a single window matching the input date range if the input date range is empty', () => { + expect( + getGranularityAlignedTimeWindows( + { + dateRange: [ + new Date('2023-01-10 00:00:30'), + new Date('2023-01-10 00:00:30'), + ], + granularity: '1 minute', + timestampValueExpression: 'TimestampTime', + } as ChartConfigWithDateRange & { granularity: string }, + [ + 60, // 1m + 5 * 60, // 5m + ], + ), + ).toEqual([ + { + dateRange: [ + new Date('2023-01-10 00:00:30'), + new Date('2023-01-10 00:00:30'), + ], + }, + ]); + }); }); describe('useQueriedChartConfig', () => { @@ -936,5 +962,36 @@ describe('useChartConfig', () => { // The original query should still have its chunked data expect(result1.current.data?.rows).toBeGreaterThan(1); }); + + it('returns an empty result if the given date range is empty', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-01 00:00:00Z'), + ], + granularity: '1 hour', + }); + + const mockResponse = createMockQueryResponse([]); + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(1); + + expect(result.current.data).toEqual({ + data: [], + meta: mockResponse.meta, + rows: 0, + isComplete: true, + }); + expect(result.current.isLoading).toBe(false); + expect(result.current.isPending).toBe(false); + }); }); }); diff --git a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx index 65483c54c..00976cc40 100644 --- a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx +++ b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx @@ -309,6 +309,40 @@ describe('useOffsetPaginatedQuery', () => { // Should have more pages available due to large time range expect(result.current.hasNextPage).toBe(true); }); + + it('should handle a time range with the same start and end date by generating one window', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T00:00:00Z'), // same start and end date + ] as [Date, Date], + }); + + // Mock the reader to return data for first window + mockReader.read + .mockResolvedValueOnce({ + done: false, + value: [ + { json: () => ['timestamp', 'message'] }, + { json: () => ['DateTime', 'String'] }, + { json: () => ['2024-01-01T01:00:00Z', 'test log 1'] }, + ], + }) + .mockResolvedValueOnce({ done: true }); + + const { result } = renderHook(() => useOffsetPaginatedQuery(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isLoading).toBe(false)); + + // Should have data from the first window + expect(result.current.data).toBeDefined(); + expect(result.current.data?.window.windowIndex).toBe(0); + + // Should have more pages available due to large time range + expect(result.current.hasNextPage).toBe(true); + }); }); describe('Pagination Within Time Windows', () => { diff --git a/packages/app/src/utils/searchWindows.ts b/packages/app/src/utils/searchWindows.ts index 7cbac8629..e7c1828a2 100644 --- a/packages/app/src/utils/searchWindows.ts +++ b/packages/app/src/utils/searchWindows.ts @@ -18,6 +18,17 @@ export function generateTimeWindowsDescending( endDate: Date, windowDurationsSeconds: number[] = DEFAULT_TIME_WINDOWS_SECONDS, ): TimeWindow[] { + if (startDate.getTime() === endDate.getTime()) { + return [ + { + startTime: startDate, + endTime: endDate, + windowIndex: 0, + direction: 'DESC', + }, + ]; + } + const windows: TimeWindow[] = []; let currentEnd = new Date(endDate); let windowIndex = 0; @@ -50,7 +61,18 @@ export function generateTimeWindowsAscending( startDate: Date, endDate: Date, windowDurationsSeconds: number[] = DEFAULT_TIME_WINDOWS_SECONDS, -) { +): TimeWindow[] { + if (startDate.getTime() === endDate.getTime()) { + return [ + { + startTime: startDate, + endTime: endDate, + windowIndex: 0, + direction: 'ASC', + }, + ]; + } + const windows: TimeWindow[] = []; let currentStart = new Date(startDate); let windowIndex = 0; From 6a210a51e9d6053a0f174d31daefd949f6575499 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 17 Oct 2025 08:21:10 -0400 Subject: [PATCH 6/6] feat: Disable chart chunking by default --- packages/app/src/components/DBTimeChart.tsx | 9 +- .../src/components/SearchTotalCountChart.tsx | 5 +- .../hooks/__tests__/useChartConfig.test.tsx | 193 +++++++++++++++--- packages/app/src/hooks/useChartConfig.tsx | 43 ++-- packages/app/src/hooks/usePatterns.tsx | 6 +- 5 files changed, 195 insertions(+), 61 deletions(-) diff --git a/packages/app/src/components/DBTimeChart.tsx b/packages/app/src/components/DBTimeChart.tsx index dec94fc3f..10a0410fb 100644 --- a/packages/app/src/components/DBTimeChart.tsx +++ b/packages/app/src/components/DBTimeChart.tsx @@ -62,11 +62,12 @@ function DBTimeChartComponent({ limit: { limit: 100000 }, }; - const { data, isLoading, isError, error, isSuccess, isFetching } = + const { data, isLoading, isError, error, isPlaceholderData, isSuccess } = useQueriedChartConfig(queriedConfig, { placeholderData: (prev: any) => prev, - queryKey: [queryKeyPrefix, queriedConfig], + queryKey: [queryKeyPrefix, queriedConfig, 'chunked'], enabled, + enableQueryChunking: true, }); useEffect(() => { @@ -75,6 +76,8 @@ function DBTimeChartComponent({ } }, [isError, isErrorExpanded, errorExpansion]); + const isLoadingOrPlaceholder = + isLoading || !data?.isComplete || isPlaceholderData; const { data: source } = useSource({ id: sourceId }); const { graphResults, timestampColumn, groupKeys, lineNames, lineColors } = @@ -337,7 +340,7 @@ function DBTimeChartComponent({ graphResults={graphResults} groupKeys={groupKeys} isClickActive={false} - isLoading={isFetching} + isLoading={isLoadingOrPlaceholder} lineColors={lineColors} lineNames={lineNames} logReferenceTimestamp={logReferenceTimestamp} diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index a6da697c8..e1a02d4bd 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -10,7 +10,7 @@ export function useSearchTotalCount( config: ChartConfigWithDateRange, queryKeyPrefix: string, ) { - // copied from DBTimeChart + // queriedConfig, queryKey, and enableQueryChunking match DBTimeChart so that react query can de-dupe these queries. const { granularity } = useTimeChartSettings(config); const queriedConfig = { ...config, @@ -22,10 +22,11 @@ export function useSearchTotalCount( isLoading, isError, } = useQueriedChartConfig(queriedConfig, { - queryKey: [queryKeyPrefix, queriedConfig], + queryKey: [queryKeyPrefix, queriedConfig, 'chunked'], staleTime: 1000 * 60 * 5, refetchOnWindowFocus: false, placeholderData: keepPreviousData, // no need to flash loading state when in live tail + enableQueryChunking: true, }); const isTotalCountComplete = !!totalCountData?.isComplete; diff --git a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx index cc66edc75..fbf1c431f 100644 --- a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx +++ b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx @@ -364,9 +364,12 @@ describe('useChartConfig', () => { mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); await waitFor(() => expect(result.current.isSuccess).toBe(true)); await waitFor(() => expect(result.current.isFetching).toBe(false)); @@ -410,9 +413,12 @@ describe('useChartConfig', () => { mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); await waitFor(() => expect(result.current.isSuccess).toBe(true)); await waitFor(() => expect(result.current.isFetching).toBe(false)); @@ -460,9 +466,12 @@ describe('useChartConfig', () => { mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); await waitFor(() => expect(result.current.isSuccess).toBe(true)); await waitFor(() => expect(result.current.isFetching).toBe(false)); @@ -536,9 +545,12 @@ describe('useChartConfig', () => { mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); await waitFor(() => expect(result.current.isSuccess).toBe(true)); await waitFor(() => expect(result.current.isFetching).toBe(false)); @@ -560,7 +572,7 @@ describe('useChartConfig', () => { }); }); - it('fetches data without chunking when disableQueryChunking is true', async () => { + it('fetches data without chunking when enableQueryChunking is false', async () => { const config = createMockChartConfig({ dateRange: [ new Date('2025-10-01 00:00:00Z'), @@ -585,7 +597,7 @@ describe('useChartConfig', () => { mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); const { result } = renderHook( - () => useQueriedChartConfig(config, { disableQueryChunking: true }), + () => useQueriedChartConfig(config, { enableQueryChunking: false }), { wrapper, }, @@ -611,6 +623,54 @@ describe('useChartConfig', () => { }); }); + it('fetches data without chunking when enableQueryChunking is not provided', async () => { + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '1 hour', + }); + + const mockResponse = createMockQueryResponse([ + { + 'count()': '71', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-01T00:00:00Z', + }, + { + 'count()': '73', + SeverityText: 'info', + __hdx_time_bucket: '2025-10-02T00:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); + + const { result } = renderHook(() => useQueriedChartConfig(config), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + // Should only be called once since chunking is explicitly disabled + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledTimes(1); + expect(mockClickhouseClient.queryChartConfig).toHaveBeenCalledWith({ + config, + metadata: expect.any(Object), + opts: { + abort_signal: expect.any(AbortSignal), + }, + }); + expect(result.current.data).toEqual({ + data: mockResponse.data, + meta: mockResponse.meta, + rows: mockResponse.rows, + isComplete: true, + }); + }); + it('fetches data with chunking when granularity and date range are provided', async () => { const config = createMockChartConfig({ dateRange: [ @@ -654,9 +714,12 @@ describe('useChartConfig', () => { .mockResolvedValueOnce(mockResponse2) .mockResolvedValueOnce(mockResponse3); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); await waitFor(() => expect(result.current.isSuccess).toBe(true)); await waitFor(() => expect(result.current.isFetching).toBe(false)); @@ -746,9 +809,12 @@ describe('useChartConfig', () => { .mockResolvedValueOnce(mockResponse2) .mockResolvedValueOnce(mockResponse3); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); await waitFor(() => expect(result.current.isSuccess).toBe(true)); await waitFor(() => expect(result.current.isPending).toBe(false)); @@ -806,9 +872,12 @@ describe('useChartConfig', () => { mockResponse1Promise, ); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); // Should be in loading state before first chunk expect(result.current.isLoading).toBe(true); @@ -850,7 +919,12 @@ describe('useChartConfig', () => { }); const { result } = renderHook( - () => useQueriedChartConfig(config, { onError, retry: false }), + () => + useQueriedChartConfig(config, { + onError, + retry: false, + enableQueryChunking: true, + }), { wrapper, }, @@ -862,6 +936,54 @@ describe('useChartConfig', () => { expect(result.current.error).toBe(mockError); }); + it('has an error status with partial data after the second chunk fails to fetch', async () => { + const mockError = new Error('Query failed'); + const mockSuccess = createMockQueryResponse([ + { + 'count()': '71', + __hdx_time_bucket: '2025-10-01T18:00:00Z', + }, + ]); + + mockClickhouseClient.queryChartConfig + .mockResolvedValueOnce(mockSuccess) + .mockRejectedValueOnce(mockError); + + const onError = jest.fn(); + const config = createMockChartConfig({ + dateRange: [ + new Date('2025-10-01 00:00:00Z'), + new Date('2025-10-02 00:00:00Z'), + ], + granularity: '3 hour', + }); + + const { result } = renderHook( + () => + useQueriedChartConfig(config, { + onError, + retry: false, + enableQueryChunking: true, + }), + { + wrapper, + }, + ); + + await waitFor(() => expect(result.current.isError).toBe(true)); + expect(result.current.isFetching).toBe(false); + expect(result.current.isSuccess).toBe(false); + + expect(result.current.error).toBe(mockError); + expect(onError).toHaveBeenCalledWith(mockError); + expect(result.current.data).toEqual({ + data: mockSuccess.data, + meta: mockSuccess.meta, + rows: mockSuccess.rows, + isComplete: false, + }); + }); + it('does not make requests if it is disabled', async () => { const config = createMockChartConfig({ dateRange: [ @@ -881,7 +1003,11 @@ describe('useChartConfig', () => { mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); const { result } = renderHook( - () => useQueriedChartConfig(config, { enabled: false }), + () => + useQueriedChartConfig(config, { + enabled: false, + enableQueryChunking: true, + }), { wrapper, }, @@ -895,7 +1021,7 @@ describe('useChartConfig', () => { expect(result.current.data).toBeUndefined(); }); - it('uses different query keys for the same config when one sets disableQueryChunking', async () => { + it('uses different query keys for the same config when one sets enableQueryChunking', async () => { const config = createMockChartConfig({ dateRange: [ new Date('2025-10-01 00:00:00Z'), @@ -923,7 +1049,7 @@ describe('useChartConfig', () => { ); const { result: result1 } = renderHook( - () => useQueriedChartConfig(config), + () => useQueriedChartConfig(config, { enableQueryChunking: true }), { wrapper, }, @@ -938,13 +1064,13 @@ describe('useChartConfig', () => { expect(chunkedCallCount).toBeGreaterThan(1); expect(result1.current.data?.rows).toBeGreaterThan(1); - // Second render with same config but disableQueryChunking=true + // Second render with same config but without query chunking enabled mockClickhouseClient.queryChartConfig.mockResolvedValue( mockResponseNonChunked, ); const { result: result2 } = renderHook( - () => useQueriedChartConfig(config, { disableQueryChunking: true }), + () => useQueriedChartConfig(config), { wrapper, }, @@ -975,9 +1101,12 @@ describe('useChartConfig', () => { const mockResponse = createMockQueryResponse([]); mockClickhouseClient.queryChartConfig.mockResolvedValue(mockResponse); - const { result } = renderHook(() => useQueriedChartConfig(config), { - wrapper, - }); + const { result } = renderHook( + () => useQueriedChartConfig(config, { enableQueryChunking: true }), + { + wrapper, + }, + ); await waitFor(() => expect(result.current.isSuccess).toBe(true)); await waitFor(() => expect(result.current.isFetching).toBe(false)); diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index 689874685..53cdf3f55 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -35,12 +35,12 @@ import { generateTimeWindowsDescending } from '@/utils/searchWindows'; interface AdditionalUseQueriedChartConfigOptions { onError?: (error: Error | ClickHouseQueryError) => void; /** - * By default, queries with large date ranges are split into multiple smaller queries to + * Queries with large date ranges can be split into multiple smaller queries to * avoid overloading the ClickHouse server and running into timeouts. In some cases, such * as when data is being sampled across the entire range, this chunking is not desirable - * and can be disabled. + * and should be disabled. */ - disableQueryChunking?: boolean; + enableQueryChunking?: boolean; } type TimeWindow = { @@ -121,16 +121,21 @@ export const getGranularityAlignedTimeWindows = ( return windows; }; -async function* fetchDataInChunks( - config: ChartConfigWithOptDateRange, - clickhouseClient: ClickhouseClient, - signal: AbortSignal, - disableQueryChunking: boolean = false, -) { +async function* fetchDataInChunks({ + config, + clickhouseClient, + signal, + enableQueryChunking = false, +}: { + config: ChartConfigWithOptDateRange; + clickhouseClient: ClickhouseClient; + signal: AbortSignal; + enableQueryChunking?: boolean; +}) { const windows = - !disableQueryChunking && shouldUseChunking(config) + enableQueryChunking && shouldUseChunking(config) ? getGranularityAlignedTimeWindows(config) - : ([undefined] as const); + : [undefined]; if (IS_MTVIEWS_ENABLED) { const { dataTableDDL, mtViewDDL, renderMTViewConfig } = @@ -181,7 +186,7 @@ function appendChunk( * * If all of the following are true, the query will be chunked into multiple smaller queries: * - The config includes a dateRange, granularity, and timestampValueExpression - * - `options.disableQueryChunking` is falsy + * - `options.enableQueryChunking` is true * * For chunked queries, note the following: * - `config.limit`, if provided, is applied to each chunk, so the total number @@ -202,9 +207,9 @@ export function useQueriedChartConfig( const queryClient = useQueryClient(); const query = useQuery({ - // Include disableQueryChunking in the query key to ensure that queries with the - // same config but different disableQueryChunking values do not share a query - queryKey: [config, options?.disableQueryChunking ?? false], + // Include enableQueryChunking in the query key to ensure that queries with the + // same config but different enableQueryChunking values do not share a query + queryKey: [config, options?.enableQueryChunking ?? false], // TODO: Replace this with `streamedQuery` when it is no longer experimental. Use 'replace' refetch mode. // https://tanstack.com/query/latest/docs/reference/streamedQuery queryFn: async context => { @@ -220,12 +225,12 @@ export function useQueriedChartConfig( isComplete: false, }; - const chunks = fetchDataInChunks( + const chunks = fetchDataInChunks({ config, clickhouseClient, - context.signal, - options?.disableQueryChunking, - ); + signal: context.signal, + enableQueryChunking: options?.enableQueryChunking, + }); let accumulatedChunks: TQueryFnData = emptyValue; for await (const chunk of chunks) { diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index 43f956605..ea8528850 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -144,11 +144,7 @@ function usePatterns({ const { data: sampleRows, isLoading: isSampleLoading } = useQueriedChartConfig( configWithPrimaryAndPartitionKey ?? config, // `config` satisfying type, never used due to `enabled` check - { - enabled: configWithPrimaryAndPartitionKey != null && enabled, - // Disable chunking to ensure we get the desired sample size - disableQueryChunking: true, - }, + { enabled: configWithPrimaryAndPartitionKey != null && enabled }, ); const { data: pyodide, isLoading: isLoadingPyodide } = usePyodide({