From 9e7aa1631c1b6dcd450c02adf42527ccaabeab24 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 6 Aug 2024 11:11:51 -0500 Subject: [PATCH 01/37] chore: refactor to allow multiple queries --- .../Breakdowns/FieldsBreakdownScene.tsx | 4 +- .../Breakdowns/LabelBreakdownScene.tsx | 8 ++-- .../Breakdowns/PatternsLogsSampleScene.tsx | 4 +- .../ServiceScene/LogsVolumePanel.tsx | 6 +-- src/Components/ServiceScene/ServiceScene.tsx | 4 +- .../ServiceSelectionScene.tsx | 20 ++++---- src/services/datasource.ts | 47 ++++++++++++++++++- src/services/panel.ts | 11 +++-- 8 files changed, 78 insertions(+), 26 deletions(-) diff --git a/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx index 08cce9a33..2036b5e56 100644 --- a/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx @@ -287,7 +287,7 @@ export class FieldsBreakdownScene extends SceneObjectBase this.state.search.state.filter ?? ''; return new LayoutSwitcher({ - $data: getQueryRunner(query), + $data: getQueryRunner([query]), options: [ { value: 'single', label: 'Single' }, { value: 'grid', label: 'Grid' }, diff --git a/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx index 416c33807..bdd0d0e2b 100644 --- a/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx @@ -218,9 +218,9 @@ export class LabelBreakdownScene extends SceneObjectBase this.state.search.state.filter ?? ''; return new LayoutSwitcher({ - $data: getQueryRunner(query), + $data: getQueryRunner([query]), options: [ { value: 'single', label: 'Single' }, { value: 'grid', label: 'Grid' }, diff --git a/src/Components/ServiceScene/Breakdowns/PatternsLogsSampleScene.tsx b/src/Components/ServiceScene/Breakdowns/PatternsLogsSampleScene.tsx index ddc545f5e..24c1e230b 100644 --- a/src/Components/ServiceScene/Breakdowns/PatternsLogsSampleScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/PatternsLogsSampleScene.tsx @@ -48,7 +48,7 @@ export class PatternsLogsSampleScene extends SceneObjectBase { this.onQueryWithFiltersResult(value); }); @@ -198,7 +198,7 @@ export class PatternsLogsSampleScene extends SceneObjectBase { .setOption('legend', { showLegend: true, calcs: ['sum'], displayMode: LegendDisplayMode.List }) .setUnit('short') .setData( - getQueryRunner( + getQueryRunner([ buildDataQuery(getTimeSeriesExpr(this, LEVEL_VARIABLE_VALUE, false), { legendFormat: `{{${LEVEL_VARIABLE_VALUE}}}`, - }) - ) + }), + ]) ) .setCustomFieldConfig('stacking', { mode: StackingMode.Normal }) .setCustomFieldConfig('fillOpacity', 100) diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index bf50b2a41..da68c9bd9 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -90,7 +90,7 @@ export class ServiceScene extends SceneObjectBase { public constructor(state: MakeOptional) { super({ body: state.body ?? buildGraphScene(), - $data: getQueryRunner(buildDataQuery(LOG_STREAM_SELECTOR_EXPR)), + $data: getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR)]), loading: true, ...state, }); @@ -175,7 +175,7 @@ export class ServiceScene extends SceneObjectBase { let stateUpdate: Partial = {}; if (!this.state.$data) { - stateUpdate.$data = getQueryRunner(buildDataQuery(LOG_STREAM_SELECTOR_EXPR)); + stateUpdate.$data = getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR)]); } if (!this.state.body) { diff --git a/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx b/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx index 345bd1546..36ab6ddfa 100644 --- a/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx +++ b/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx @@ -88,7 +88,7 @@ export class ServiceSelectionScene extends SceneObjectBase(), ...state, }); @@ -215,13 +215,13 @@ export class ServiceSelectionScene extends SceneObjectBase {/** When services fetched, show how many services are we showing */} {isLogVolumeLoading && } - {!isLogVolumeLoading && <>Showing {serviceCount} service{serviceCount > 1 ? 's' : ''}} + {!isLogVolumeLoading && ( + <> + Showing {serviceCount} service{serviceCount > 1 ? 's' : ''} + + )} { } case 'patterns': { console.warn('not yet implemented'); - // this.transformPatternResponse(request, ds, subscriber); + this.getPatterns(request, ds, subscriber); break; } default: { @@ -79,6 +79,51 @@ class WrappedLokiDatasource extends RuntimeDataSource { }); } + private getPatterns( + request: DataQueryRequest, + ds: DataSourceWithBackend, + subscriber: Subscriber + ) { + if (request.targets.length !== 1) { + throw new Error('Patterns query can only have a single target!'); + } + + const targetsInterpolated = ds.interpolateVariablesInQueries(request.targets, request.scopedVars); + const expression = targetsInterpolated[0].expr; + + const dsResponse = ds.getResource( + 'patterns', + { + query: expression, + from: request.range.from.utc().toISOString(), + to: request.range.to.utc().toISOString(), + limit: 1000, + }, + { + requestId: request.requestId ?? 'volume', + headers: { + 'X-Query-Tags': `Source=${PLUGIN_ID}`, + }, + } + ); + dsResponse.then((response: IndexVolumeResponse | undefined) => { + response?.data.result.sort((lhs: VolumeResult, rhs: VolumeResult) => { + const lVolumeCount: VolumeCount = lhs.value[1]; + const rVolumeCount: VolumeCount = rhs.value[1]; + return Number(rVolumeCount) - Number(lVolumeCount); + }); + // Scenes will only emit dataframes from the SceneQueryRunner, so for now we need to convert the API response to a dataframe + const df = createDataFrame({ + fields: [ + { name: 'service_name', values: response?.data.result.map((r) => r.metric.service_name) }, + { name: 'volume', values: response?.data.result.map((r) => Number(r.value[1])) }, + ], + }); + subscriber.next({ data: [df] }); + subscriber.complete(); + }); + } + private getVolume( request: DataQueryRequest, ds: DataSourceWithBackend, diff --git a/src/services/panel.ts b/src/services/panel.ts index ef83932c4..16a5faddb 100644 --- a/src/services/panel.ts +++ b/src/services/panel.ts @@ -78,16 +78,19 @@ export function sortLevelTransformation() { }; } -export function getQueryRunner(query: LokiQuery) { +export function getQueryRunner(queries: LokiQuery[]) { // if there's a legendFormat related to any `level` like label, we want to // sort the output equally. That's purposefully not `LEVEL_VARIABLE_VALUE`, // such that the `detected_level` graph looks the same as a graph for the // `level` label. - if (query.legendFormat?.toLowerCase().includes('level')) { + + const hasLevel = queries.find((query) => query.legendFormat?.toLowerCase().includes('level')); + + if (hasLevel) { return new SceneDataTransformer({ $data: new SceneQueryRunner({ datasource: { uid: WRAPPED_LOKI_DS_UID }, - queries: [query], + queries: queries, }), transformations: [sortLevelTransformation], }); @@ -95,6 +98,6 @@ export function getQueryRunner(query: LokiQuery) { return new SceneQueryRunner({ datasource: { uid: WRAPPED_LOKI_DS_UID }, - queries: [query], + queries: queries, }); } From d55c9ac770450750be1a8b449e1d9b5c3aebaa95 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 6 Aug 2024 16:42:06 -0500 Subject: [PATCH 02/37] chore: wip - major functionality implemented --- .../Patterns/PatternsBreakdownScene.tsx | 91 +++----- .../Patterns/PatternsFrameScene.tsx | 16 +- .../ServiceScene/LogsTableScene.tsx | 4 +- src/Components/ServiceScene/ServiceScene.tsx | 162 ++++++++------ src/Components/Table/TableProvider.tsx | 2 +- src/services/datasource.ts | 210 +++++++++++++----- src/services/metadata.ts | 2 +- src/services/navigate.test.ts | 18 +- src/services/query.ts | 1 + 9 files changed, 311 insertions(+), 195 deletions(-) diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index 4c2954e72..e62c2f833 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -1,7 +1,7 @@ import { css } from '@emotion/css'; import React from 'react'; -import { DataFrame, dateTime, FieldType, GrafanaTheme2 } from '@grafana/data'; +import { DataFrame, dateTime, GrafanaTheme2 } from '@grafana/data'; import { CustomVariable, SceneComponentProps, @@ -15,7 +15,7 @@ import { import { Text, useStyles2 } from '@grafana/ui'; import { StatusWrapper } from 'Components/ServiceScene/Breakdowns/StatusWrapper'; import { VAR_LABEL_GROUP_BY } from 'services/variables'; -import { LokiPattern, ServiceScene } from '../../ServiceScene'; +import { getPatternsFrames, ServiceScene } from '../../ServiceScene'; import { IndexScene } from '../../../IndexScene/IndexScene'; import { PatternsFrameScene } from './PatternsFrameScene'; import { PatternsViewTextSearch } from './PatternsViewTextSearch'; @@ -53,8 +53,8 @@ export class PatternsBreakdownScene extends SceneObjectBase= PATTERNS_MAX_AGE_HOURS; @@ -101,16 +102,21 @@ export class PatternsBreakdownScene extends SceneObjectBase { - if (!areArraysEqual(newState.patterns, prevState.patterns)) { - this.updatePatternFrames(newState.patterns); + const newFrame = getPatternsFrames(newState?.$data?.state?.data); + const prevFrame = getPatternsFrames(prevState?.$data?.state?.data); + console.log('newFrame', newFrame); + if (!areArraysEqual(newFrame, prevFrame)) { + this.updatePatternFrames(newFrame); } }) ); @@ -133,7 +139,8 @@ export class PatternsBreakdownScene extends SceneObjectBase { - const timeValues: number[] = []; - const sampleValues: number[] = []; - let sum = 0; - pat.samples.forEach(([time, value]) => { - timeValues.push(time * 1000); - const sample = parseFloat(value); - sampleValues.push(sample); - if (sample > maxValue) { - maxValue = sample; - } - if (sample < minValue) { - minValue = sample; - } - sum += sample; - }); - const dataFrame: DataFrame = { - refId: pat.pattern, - fields: [ - { - name: 'time', - type: FieldType.time, - values: timeValues, - config: {}, - }, - { - name: pat.pattern, - type: FieldType.number, - values: sampleValues, - config: {}, - }, - ], - length: pat.samples.length, - meta: { - preferredVisualisationType: 'graph', - }, - }; - const existingPattern = appliedPatterns?.find((appliedPattern) => appliedPattern.pattern === pat.pattern); - - return { - dataFrame, - pattern: pat.pattern, - sum, - status: existingPattern?.type, - }; - }) - .sort((a, b) => b.sum - a.sum); + return patterns.map((dataFrame) => { + const existingPattern = appliedPatterns?.find((appliedPattern) => appliedPattern.pattern === dataFrame.name); + + const sum: number = dataFrame.meta?.custom?.sum; + const patternFrame: PatternFrame = { + dataFrame, + pattern: dataFrame.name as string, + sum, + status: existingPattern?.type, + }; + + return patternFrame; + }); } } diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx index b15d54284..6fcbe55d3 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx @@ -12,7 +12,7 @@ import { VizPanel, } from '@grafana/scenes'; import { LegendDisplayMode, PanelContext, SeriesVisibilityChangeMode } from '@grafana/ui'; -import { ServiceScene } from '../../ServiceScene'; +import { getPatternsFrames, ServiceScene } from '../../ServiceScene'; import { onPatternClick } from './FilterByPatternsButton'; import { IndexScene } from '../../../IndexScene/IndexScene'; import { PatternsViewTableScene } from './PatternsViewTableScene'; @@ -44,7 +44,9 @@ export class PatternsFrameScene extends SceneObjectBase public static Component = ({ model }: SceneComponentProps) => { const { body, loading } = model.useState(); const logsByServiceScene = sceneGraph.getAncestor(model, ServiceScene); - const { patterns } = logsByServiceScene.useState(); + const { $data } = logsByServiceScene.useState(); + const patterns = getPatternsFrames($data?.state.data); + return (
{!loading && patterns && patterns.length > 0 && <>{body && }} @@ -58,7 +60,10 @@ export class PatternsFrameScene extends SceneObjectBase // If the patterns have changed, recalculate the dataframes this._subs.add( sceneGraph.getAncestor(this, ServiceScene).subscribeToState((newState, prevState) => { - if (!areArraysEqual(newState.patterns, prevState.patterns)) { + const newFrame = getPatternsFrames(newState?.$data?.state?.data); + const prevFrame = getPatternsFrames(prevState?.$data?.state?.data); + + if (!areArraysEqual(newFrame, prevFrame)) { const patternsBreakdownScene = sceneGraph.getAncestor(this, PatternsBreakdownScene); this.updatePatterns(patternsBreakdownScene.state.patternFrames); @@ -107,12 +112,15 @@ export class PatternsFrameScene extends SceneObjectBase const patternFrames = patternsBreakdownScene.state.patternFrames; const serviceScene = sceneGraph.getAncestor(this, ServiceScene); - const lokiPatterns = serviceScene.state.patterns; + + const lokiPatterns = getPatternsFrames(serviceScene.state.$data?.state.data); if (!lokiPatterns || !patternFrames) { console.warn('Failed to update PatternsFrameScene body'); return; } + console.log('patterns update body', lokiPatterns); + this.setState({ body: this.getSingleViewLayout(), legendSyncPatterns: new Set(), diff --git a/src/Components/ServiceScene/LogsTableScene.tsx b/src/Components/ServiceScene/LogsTableScene.tsx index 3621a76c7..b3f8f4432 100644 --- a/src/Components/ServiceScene/LogsTableScene.tsx +++ b/src/Components/ServiceScene/LogsTableScene.tsx @@ -8,6 +8,7 @@ import { LogsPanelHeaderActions } from '../Table/LogsHeaderActions'; import { css } from '@emotion/css'; import { addAdHocFilter } from './Breakdowns/AddToFiltersButton'; import { areArraysEqual } from '../../services/comparison'; +import { getLogsPanelFrame } from './ServiceScene'; export class LogsTableScene extends SceneObjectBase { public static Component = ({ model }: SceneComponentProps) => { @@ -44,6 +45,7 @@ export class LogsTableScene extends SceneObjectBase { }; const styles = getStyles(); + const dataFrame = getLogsPanelFrame(data); return (
@@ -60,7 +62,7 @@ export class LogsTableScene extends SceneObjectBase { selectedLine={selectedLine} urlColumns={urlColumns ?? []} setUrlColumns={setUrlColumns} - dataFrame={data?.series[0]} + dataFrame={dataFrame} clearSelectedLine={clearSelectedLine} /> )} diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index da68c9bd9..177c539f2 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -1,7 +1,7 @@ import { css } from '@emotion/css'; import React from 'react'; -import { GrafanaTheme2, LoadingState } from '@grafana/data'; +import { GrafanaTheme2, PanelData } from '@grafana/data'; import { SceneComponentProps, SceneFlexItem, @@ -17,18 +17,18 @@ import { Box, LoadingPlaceholder, Stack, Tab, TabsBar, useStyles2 } from '@grafa import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; import { DetectedLabel, DetectedLabelsResponse, updateParserFromDataFrame } from 'services/fields'; import { getQueryRunner } from 'services/panel'; -import { buildDataQuery, renderLogQLStreamSelector } from 'services/query'; +import { buildDataQuery, buildResourceQuery } from 'services/query'; import { getDrilldownSlug, getDrilldownValueSlug, PageSlugs, PLUGIN_ID, ValueSlugs } from 'services/routing'; import { getExplorationFor, getLokiDatasource } from 'services/scenes'; import { ALL_VARIABLE_VALUE, - getFieldsVariable, getLabelsVariable, LEVEL_VARIABLE_VALUE, LOG_STREAM_SELECTOR_EXPR, VAR_DATASOURCE, VAR_FIELDS, VAR_LABELS, + VAR_LABELS_EXPR, VAR_LEVELS, VAR_PATTERNS, } from 'services/variables'; @@ -47,11 +47,6 @@ import { getMetadataService } from '../../services/metadata'; import { navigateToDrilldownPage, navigateToIndex } from '../../services/navigate'; import { areArraysEqual } from '../../services/comparison'; -export interface LokiPattern { - pattern: string; - samples: Array<[number, string]>; -} - interface BreakdownViewDefinition { displayName: string; value: PageSlugs; @@ -71,7 +66,7 @@ type MakeOptional = Pick, K> & Omit; export interface ServiceSceneCustomState { fields?: string[]; labels?: DetectedLabel[]; - patterns?: LokiPattern[]; + // patterns?: LokiPattern[]; fieldsCount?: number; loading?: boolean; } @@ -81,6 +76,24 @@ export interface ServiceSceneState extends SceneObjectState, ServiceSceneCustomS drillDownLabel?: string; } +const LOGS_PANEL_QUERY_REFID = 'logsPanelQuery'; +const PATTERNS_QUERY_REFID = 'patterns'; + +function getServiceSceneQueryRunner() { + return getQueryRunner([ + buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID }), + buildResourceQuery(VAR_LABELS_EXPR, 'patterns', { refId: PATTERNS_QUERY_REFID }), + ]); +} + +export function getLogsPanelFrame(data: PanelData | undefined) { + return data?.series.find((series) => series.refId === LOGS_PANEL_QUERY_REFID); +} + +export function getPatternsFrames(data: PanelData | undefined) { + return data?.series.filter((series) => series.refId === PATTERNS_QUERY_REFID); +} + export class ServiceScene extends SceneObjectBase { protected _variableDependency = new VariableDependencyConfig(this, { variableNames: [VAR_DATASOURCE, VAR_LABELS, VAR_FIELDS, VAR_PATTERNS, VAR_LEVELS], @@ -88,9 +101,10 @@ export class ServiceScene extends SceneObjectBase { }); public constructor(state: MakeOptional) { + console.log('service scene constructor'); super({ body: state.body ?? buildGraphScene(), - $data: getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR)]), + $data: getServiceSceneQueryRunner(), loading: true, ...state, }); @@ -153,20 +167,36 @@ export class ServiceScene extends SceneObjectBase { if (this.state.$data) { this._subs.add( this.state.$data?.subscribeToState((newState) => { - if (newState.data?.state === LoadingState.Done) { + // console.log('data change', newState) + const logsPanelResponse = getLogsPanelFrame(newState.data); + const patternsResponse = getPatternsFrames(newState.data); + // console.log('$data change', newState) + if (logsPanelResponse) { this.updateFields(); + } else if (patternsResponse) { + console.log('patterns response', patternsResponse); + + // @todo fix + // this.setState({ + // patterns: patternsResponse.fields.map(pattern => { + // return { + // pattern: pattern.name, + // samples: pattern.values + // } + // }) + // }) } }) ); } this.updateLabels(); - this.updatePatterns(); + // this.updatePatterns(); this._subs.add( sceneGraph.getTimeRange(this).subscribeToState(() => { this.updateLabels(); - this.updatePatterns(); + // this.updatePatterns(); }) ); } @@ -175,7 +205,7 @@ export class ServiceScene extends SceneObjectBase { let stateUpdate: Partial = {}; if (!this.state.$data) { - stateUpdate.$data = getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR)]); + stateUpdate.$data = getServiceSceneQueryRunner(); } if (!this.state.body) { @@ -197,7 +227,7 @@ export class ServiceScene extends SceneObjectBase { if (filterVariable.state.filters.length === 0) { return; } - Promise.all([this.updatePatterns(), this.updateLabels()]) + Promise.all([this.updateLabels()]) .finally(() => { // For patterns, we don't want to reload to logs as we allow users to select multiple patterns if (variable.state.name !== VAR_PATTERNS) { @@ -227,24 +257,19 @@ export class ServiceScene extends SceneObjectBase { 'user_identifier', ]; const newState = sceneGraph.getData(this).state; - if (newState.data?.state === LoadingState.Done) { - const frame = newState.data?.series[0]; - if (frame) { - const res = updateParserFromDataFrame(frame, this); - const fields = res.fields.filter((f) => !disabledFields.includes(f)).sort((a, b) => a.localeCompare(b)); - if (!areArraysEqual(fields, this.state.fields)) { - this.setState({ - fields: fields, - loading: false, - }); - } - } else { + const frame = getLogsPanelFrame(newState.data); + if (frame) { + const res = updateParserFromDataFrame(frame, this); + const fields = res.fields.filter((f) => !disabledFields.includes(f)).sort((a, b) => a.localeCompare(b)); + // console.log('fields', fields) + if (!areArraysEqual(fields, this.state.fields)) { this.setState({ - fields: [], + fields: fields, loading: false, }); } - } else if (newState.data?.state === LoadingState.Error) { + } else { + // console.log('clearing fields', newState.data) this.setState({ fields: [], loading: false, @@ -252,42 +277,42 @@ export class ServiceScene extends SceneObjectBase { } } - private async updatePatterns() { - const ds = await getLokiDatasource(this); - if (!ds) { - return; - } - - const timeRange = sceneGraph.getTimeRange(this).state.value; - const labels = getLabelsVariable(this); - const fields = getFieldsVariable(this); - - const excludeLabels = [ALL_VARIABLE_VALUE, LEVEL_VARIABLE_VALUE]; - - const { data } = await ds.getResource( - 'patterns', - { - query: renderLogQLStreamSelector([ - // this will only be the service name for now - ...labels.state.filters, - // only include fields that are an indexed label - ...fields.state.filters.filter( - // we manually add level as a label, but it'll be structured metadata mostly, so we skip it here - (field) => - this.state.labels?.find((label) => label.label === field.key) && !excludeLabels.includes(field.key) - ), - ]), - start: timeRange.from.utc().toISOString(), - end: timeRange.to.utc().toISOString(), - }, - { - headers: { - 'X-Query-Tags': `Source=${PLUGIN_ID}`, - }, - } - ); - this.setState({ patterns: data }); - } + // private async updatePatterns() { + // const ds = await getLokiDatasource(this); + // if (!ds) { + // return; + // } + // + // const timeRange = sceneGraph.getTimeRange(this).state.value; + // const labels = getLabelsVariable(this); + // const fields = getFieldsVariable(this); + // + // const excludeLabels = [ALL_VARIABLE_VALUE, LEVEL_VARIABLE_VALUE]; + // + // const { data } = await ds.getResource( + // 'patterns', + // { + // query: renderLogQLStreamSelector([ + // // this will only be the service name for now + // ...labels.state.filters, + // // only include fields that are an indexed label + // ...fields.state.filters.filter( + // // we manually add level as a label, but it'll be structured metadata mostly, so we skip it here + // (field) => + // this.state.labels?.find((label) => label.label === field.key) && !excludeLabels.includes(field.key) + // ), + // ]), + // start: timeRange.from.utc().toISOString(), + // end: timeRange.to.utc().toISOString(), + // }, + // { + // headers: { + // 'X-Query-Tags': `Source=${PLUGIN_ID}`, + // }, + // } + // ); + // this.setState({ patterns: data }); + // } private async updateLabels() { const ds = await getLokiDatasource(this); @@ -433,12 +458,15 @@ export class LogsActionBar extends SceneObjectBase { } } + const serviceScene = sceneGraph.getAncestor(model, ServiceScene); + const { loading, $data, ...state } = serviceScene.useState(); + const getCounter = (tab: BreakdownViewDefinition, state: ServiceSceneState) => { switch (tab.value) { case 'fields': return state.fieldsCount ?? (state.fields?.filter((l) => l !== ALL_VARIABLE_VALUE) ?? []).length; case 'patterns': - return state.patterns?.length; + return getPatternsFrames($data?.state.data)?.length ?? 0; case 'labels': return (state.labels?.filter((l) => l.label !== ALL_VARIABLE_VALUE) ?? []).length; default: @@ -446,8 +474,6 @@ export class LogsActionBar extends SceneObjectBase { } }; - const serviceScene = sceneGraph.getAncestor(model, ServiceScene); - const { loading, ...state } = serviceScene.useState(); return (
diff --git a/src/Components/Table/TableProvider.tsx b/src/Components/Table/TableProvider.tsx index 7066ba967..da12ff331 100644 --- a/src/Components/Table/TableProvider.tsx +++ b/src/Components/Table/TableProvider.tsx @@ -7,7 +7,7 @@ import { parseLogsFrame } from '../../services/logsFrame'; import { SelectedTableRow } from './LogLineCellComponent'; interface TableProviderProps { - dataFrame: DataFrame; + dataFrame?: DataFrame; setUrlColumns: (columns: string[]) => void; urlColumns: string[]; addFilter: (filter: AdHocVariableFilter) => void; diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 217695193..3548e34f8 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -1,8 +1,15 @@ -import { createDataFrame, DataQueryRequest, DataQueryResponse, TestDataSourceResponse } from '@grafana/data'; +import { + createDataFrame, + DataQueryRequest, + DataQueryResponse, + FieldType, + LoadingState, + TestDataSourceResponse, +} from '@grafana/data'; import { DataSourceWithBackend, getDataSourceSrv } from '@grafana/runtime'; import { RuntimeDataSource, SceneObject, sceneUtils } from '@grafana/scenes'; import { DataQuery } from '@grafana/schema'; -import { Observable, Subscriber } from 'rxjs'; +import { Observable, Subscriber, tap } from 'rxjs'; import { getDataSource } from './scenes'; import { LokiQuery } from './query'; import { PLUGIN_ID } from './routing'; @@ -32,13 +39,29 @@ type IndexVolumeResponse = { }; }; +type SampleTimeStamp = number; +type SampleCount = number; +type PatternSample = [SampleTimeStamp, SampleCount]; + +export interface LokiPattern { + pattern: string; + samples: PatternSample[]; +} + +type PatternsResponse = { + data: LokiPattern[]; +}; + class WrappedLokiDatasource extends RuntimeDataSource { constructor(pluginId: string, uid: string) { super(pluginId, uid); } query(request: SceneDataQueryRequest): Promise | Observable { - return new Observable((subscriber) => { + const numberOfQueries = request.targets.length; + let numberOfQueriesThatResolved = 0; + + const observable = new Observable((subscriber) => { if (!request.scopedVars?.__sceneObject) { throw new Error('Scene object not found in request'); } @@ -50,33 +73,70 @@ class WrappedLokiDatasource extends RuntimeDataSource { throw new Error('Invalid datasource!'); } - const requestType = request.targets?.[0]?.resource; + request.targets.forEach((target) => { + const requestType = target?.resource; + let newSubscriber: Subscriber; - switch (requestType) { - case 'volume': { - this.getVolume(request, ds, subscriber); - break; - } - case 'patterns': { - console.warn('not yet implemented'); - this.getPatterns(request, ds, subscriber); - break; + switch (requestType) { + case 'volume': { + newSubscriber = this.getVolume(request, ds, subscriber); + break; + } + case 'patterns': { + newSubscriber = this.getPatterns(request, ds, subscriber); + break; + } + default: { + newSubscriber = this.getData(request, ds, subscriber); + break; + } } - default: { - // override the target datasource to Loki - request.targets = request.targets.map((target) => { - target.datasource = ds; - return target; - }); - - // query the datasource and return either observable or promise - const dsResponse = ds.query(request); - dsResponse.subscribe(subscriber); - break; + + if (newSubscriber) { + subscriber.add(newSubscriber); } - } + }); }); }); + + return observable.pipe( + tap((response) => { + if (response.state === LoadingState.Done || response.state === LoadingState.Error) { + numberOfQueriesThatResolved++; + // @todo errors and cancelled queries (switch tabs while loading) + // @todo useDatasourcesFromTargets error in logs panel breaking context + if (numberOfQueriesThatResolved < numberOfQueries) { + response.state = LoadingState.Loading; + } + } + + return response; + }) + ); + } + + private getData( + request: SceneDataQueryRequest, + ds: DataSourceWithBackend, + subscriber: Subscriber + ) { + const dataQueryRequest = { ...request }; + // override the target datasource to Loki + dataQueryRequest.targets = request.targets + .filter((target) => { + return !target.resource; + }) + .map((target) => { + target.datasource = ds; + return target; + }); + + // query the datasource and return either observable or promise + const dsResponse = ds.query(dataQueryRequest); + + dsResponse.subscribe(subscriber); + + return subscriber; } private getPatterns( @@ -84,46 +144,94 @@ class WrappedLokiDatasource extends RuntimeDataSource { ds: DataSourceWithBackend, subscriber: Subscriber ) { - if (request.targets.length !== 1) { + const targets = request.targets.filter((target) => { + return target.resource === 'patterns'; + }); + + if (targets.length !== 1) { throw new Error('Patterns query can only have a single target!'); } - const targetsInterpolated = ds.interpolateVariablesInQueries(request.targets, request.scopedVars); - const expression = targetsInterpolated[0].expr; + const targetsInterpolated = ds.interpolateVariablesInQueries(targets, request.scopedVars); + const interpolatedTarget = targetsInterpolated[0]; + const expression = interpolatedTarget.expr; const dsResponse = ds.getResource( 'patterns', { query: expression, - from: request.range.from.utc().toISOString(), - to: request.range.to.utc().toISOString(), - limit: 1000, + start: request.range.from.utc().toISOString(), + end: request.range.to.utc().toISOString(), }, { - requestId: request.requestId ?? 'volume', + requestId: request.requestId ?? 'patterns', headers: { 'X-Query-Tags': `Source=${PLUGIN_ID}`, }, } ); - dsResponse.then((response: IndexVolumeResponse | undefined) => { - response?.data.result.sort((lhs: VolumeResult, rhs: VolumeResult) => { - const lVolumeCount: VolumeCount = lhs.value[1]; - const rVolumeCount: VolumeCount = rhs.value[1]; - return Number(rVolumeCount) - Number(lVolumeCount); - }); - // Scenes will only emit dataframes from the SceneQueryRunner, so for now we need to convert the API response to a dataframe - const df = createDataFrame({ - fields: [ - { name: 'service_name', values: response?.data.result.map((r) => r.metric.service_name) }, - { name: 'volume', values: response?.data.result.map((r) => Number(r.value[1])) }, - ], - }); - subscriber.next({ data: [df] }); - subscriber.complete(); + dsResponse.then((response: PatternsResponse | undefined) => { + const lokiPatterns = response?.data; + + let maxValue = -Infinity; + let minValue = 0; + + const frames = + lokiPatterns?.map((pattern: LokiPattern) => { + const timeValues: number[] = []; + const countValues: number[] = []; + let sum = 0; + pattern.samples.forEach(([time, count]) => { + timeValues.push(time * 1000); + countValues.push(count); + if (count > maxValue) { + maxValue = count; + } + if (count < minValue) { + minValue = count; + } + if (count > maxValue) { + maxValue = count; + } + if (count < minValue) { + minValue = count; + } + sum += count; + }); + return createDataFrame({ + refId: interpolatedTarget.refId, + name: pattern.pattern, + fields: [ + { + name: 'time', + type: FieldType.time, + values: timeValues, + config: {}, + }, + { + name: pattern.pattern, + type: FieldType.number, + values: countValues, + config: {}, + }, + ], + meta: { + preferredVisualisationType: 'graph', + custom: { + sum, + }, + }, + }); + }) ?? []; + + frames.sort((a, b) => (b.meta?.custom?.sum as number) - (a.meta?.custom?.sum as number)); + subscriber.next({ data: frames, state: LoadingState.Done }); }); + + return subscriber; } + //@todo doesn't work with multiple queries private getVolume( request: DataQueryRequest, ds: DataSourceWithBackend, @@ -140,8 +248,8 @@ class WrappedLokiDatasource extends RuntimeDataSource { 'index/volume', { query: expression, - from: request.range.from.utc().toISOString(), - to: request.range.to.utc().toISOString(), + start: request.range.from.utc().toISOString(), + end: request.range.to.utc().toISOString(), limit: 1000, }, { @@ -164,9 +272,11 @@ class WrappedLokiDatasource extends RuntimeDataSource { { name: 'volume', values: response?.data.result.map((r) => Number(r.value[1])) }, ], }); - subscriber.next({ data: [df] }); + subscriber.next({ data: [df], state: LoadingState.Done }); subscriber.complete(); }); + + return subscriber; } testDatasource(): Promise { diff --git a/src/services/metadata.ts b/src/services/metadata.ts index f7242cc09..708b81ec9 100644 --- a/src/services/metadata.ts +++ b/src/services/metadata.ts @@ -21,7 +21,7 @@ export class MetadataService { this.serviceSceneState = { fields: state.fields, labels: state.labels, - patterns: state.patterns, + // patterns: state.patterns, fieldsCount: state.fieldsCount, loading: state.loading, }; diff --git a/src/services/navigate.test.ts b/src/services/navigate.test.ts index 3c2748c51..9457d55de 100644 --- a/src/services/navigate.test.ts +++ b/src/services/navigate.test.ts @@ -55,15 +55,15 @@ describe('navigate', () => { mockServiceSceneState = { labels, - patterns: [ - { - pattern: 'error <_> message', - samples: [ - [1721220640, '270'], - [1721220650, '341'], - ], - }, - ], + // patterns: [ + // { + // pattern: 'error <_> message', + // samples: [ + // [1721220640, '270'], + // [1721220650, '341'], + // ], + // }, + // ], fields: ['field1', 'field2'], fieldsCount: 2, loading: true, diff --git a/src/services/query.ts b/src/services/query.ts index 10b66edcb..d05745eff 100644 --- a/src/services/query.ts +++ b/src/services/query.ts @@ -29,6 +29,7 @@ export const buildResourceQuery = ( return { ...defaultQueryParams, resource, + refId: resource, ...queryParamsOverrides, expr, }; From bc010ee47639e82dc45c66034aab8aec5b32b780 Mon Sep 17 00:00:00 2001 From: Galen Date: Wed, 7 Aug 2024 11:28:07 -0500 Subject: [PATCH 03/37] chore: refactor patterns --- src/Components/ServiceScene/BreakdownViews.ts | 65 ++++ .../Patterns/PatternsBreakdownScene.tsx | 40 ++- .../Patterns/PatternsFrameScene.tsx | 4 +- .../ServiceScene/LogsActionBarScene.tsx | 111 ++++++ src/Components/ServiceScene/ServiceScene.tsx | 330 +++++------------- src/services/datasource.ts | 26 +- src/services/metadata.ts | 2 +- 7 files changed, 301 insertions(+), 277 deletions(-) create mode 100644 src/Components/ServiceScene/BreakdownViews.ts create mode 100644 src/Components/ServiceScene/LogsActionBarScene.tsx diff --git a/src/Components/ServiceScene/BreakdownViews.ts b/src/Components/ServiceScene/BreakdownViews.ts new file mode 100644 index 000000000..e8099fb35 --- /dev/null +++ b/src/Components/ServiceScene/BreakdownViews.ts @@ -0,0 +1,65 @@ +import { PageSlugs, ValueSlugs } from '../../services/routing'; +import { buildLogsListScene } from './LogsListScene'; +import { testIds } from '../../services/testIds'; +import { buildLabelBreakdownActionScene, buildLabelValuesBreakdownActionScene } from './Breakdowns/LabelBreakdownScene'; +import { + buildFieldsBreakdownActionScene, + buildFieldValuesBreakdownActionScene, +} from './Breakdowns/FieldsBreakdownScene'; +import { buildPatternsScene } from './Breakdowns/Patterns/PatternsBreakdownScene'; +import { SceneObject } from '@grafana/scenes'; + +interface ValueBreakdownViewDefinition { + displayName: string; + value: ValueSlugs; + testId: string; + getScene: (value: string) => SceneObject; +} + +export interface BreakdownViewDefinition { + displayName: string; + value: PageSlugs; + testId: string; + getScene: (changeFields: (f: string[]) => void) => SceneObject; +} + +export const breakdownViewsDefinitions: BreakdownViewDefinition[] = [ + { + displayName: 'Logs', + value: PageSlugs.logs, + getScene: () => buildLogsListScene(), + testId: testIds.exploreServiceDetails.tabLogs, + }, + { + displayName: 'Labels', + value: PageSlugs.labels, + getScene: () => buildLabelBreakdownActionScene(), + testId: testIds.exploreServiceDetails.tabLabels, + }, + { + displayName: 'Fields', + value: PageSlugs.fields, + getScene: (f) => buildFieldsBreakdownActionScene(f), + testId: testIds.exploreServiceDetails.tabFields, + }, + { + displayName: 'Patterns', + value: PageSlugs.patterns, + getScene: () => buildPatternsScene(), + testId: testIds.exploreServiceDetails.tabPatterns, + }, +]; +export const valueBreakdownViews: ValueBreakdownViewDefinition[] = [ + { + displayName: 'Label', + value: ValueSlugs.label, + getScene: (value: string) => buildLabelValuesBreakdownActionScene(value), + testId: testIds.exploreServiceDetails.tabLabels, + }, + { + displayName: 'Field', + value: ValueSlugs.field, + getScene: (value: string) => buildFieldValuesBreakdownActionScene(value), + testId: testIds.exploreServiceDetails.tabFields, + }, +]; diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index e62c2f833..b7e355f4d 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -5,6 +5,7 @@ import { DataFrame, dateTime, GrafanaTheme2 } from '@grafana/data'; import { CustomVariable, SceneComponentProps, + SceneDataState, SceneFlexItem, SceneFlexLayout, sceneGraph, @@ -21,6 +22,7 @@ import { PatternsFrameScene } from './PatternsFrameScene'; import { PatternsViewTextSearch } from './PatternsViewTextSearch'; import { PatternsNotDetected, PatternsTooOld } from './PatternsNotDetected'; import { areArraysEqual } from '../../../../services/comparison'; +import { Unsubscribable } from 'rxjs'; export interface PatternsBreakdownSceneState extends SceneObjectState { body?: SceneFlexLayout; @@ -34,6 +36,7 @@ export interface PatternsBreakdownSceneState extends SceneObjectState { // Subset of patternFrames, undefined if empty, empty array if search results returned nothing (no data) filteredPatterns?: PatternFrame[]; patternFilter: string; + dataSub?: Unsubscribable; } export type PatternFrame = { @@ -110,18 +113,33 @@ export class PatternsBreakdownScene extends SceneObjectBase { - const newFrame = getPatternsFrames(newState?.$data?.state?.data); - const prevFrame = getPatternsFrames(prevState?.$data?.state?.data); - console.log('newFrame', newFrame); - if (!areArraysEqual(newFrame, prevFrame)) { - this.updatePatternFrames(newFrame); - } - }) - ); + const dataSub = serviceScene.state.$data.subscribeToState(this.onDataProviderChange); + this.setState({ + dataSub, + }); + this._subs.add(dataSub); + + // Subscribe to changes to the query provider + serviceScene.subscribeToState((newState, prevState) => { + if (newState.$data.state.key !== prevState.$data.state.key) { + const dataSub = serviceScene.state.$data.subscribeToState(this.onDataProviderChange); + this.state.dataSub?.unsubscribe(); + this.setState({ + dataSub, + loading: true, + }); + } + }); } + private onDataProviderChange = (newState: SceneDataState, prevState: SceneDataState) => { + const newFrame = getPatternsFrames(newState.data); + const prevFrame = getPatternsFrames(prevState.data); + if (!areArraysEqual(newFrame, prevFrame) || this.state.loading) { + this.updatePatternFrames(newFrame); + } + }; + private setBody() { this.setState({ body: new SceneFlexLayout({ @@ -140,13 +158,13 @@ export class PatternsBreakdownScene extends SceneObjectBase { constructor(state?: Partial) { super({ - loading: true, + loading: false, ...state, legendSyncPatterns: new Set(), }); @@ -119,8 +119,6 @@ export class PatternsFrameScene extends SceneObjectBase return; } - console.log('patterns update body', lokiPatterns); - this.setState({ body: this.getSingleViewLayout(), legendSyncPatterns: new Set(), diff --git a/src/Components/ServiceScene/LogsActionBarScene.tsx b/src/Components/ServiceScene/LogsActionBarScene.tsx new file mode 100644 index 000000000..857fabcde --- /dev/null +++ b/src/Components/ServiceScene/LogsActionBarScene.tsx @@ -0,0 +1,111 @@ +import { SceneComponentProps, sceneGraph, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; +import { Box, Stack, Tab, TabsBar, useStyles2 } from '@grafana/ui'; +import { getExplorationFor } from '../../services/scenes'; +import { getDrilldownSlug, getDrilldownValueSlug, PageSlugs, ValueSlugs } from '../../services/routing'; +import { GoToExploreButton } from './GoToExploreButton'; +import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '../../services/analytics'; +import { ALL_VARIABLE_VALUE, getLabelsVariable } from '../../services/variables'; +import { SERVICE_NAME } from '../ServiceSelectionScene/ServiceSelectionScene'; +import { navigateToDrilldownPage, navigateToIndex } from '../../services/navigate'; +import React from 'react'; +import { ServiceScene, ServiceSceneState } from './ServiceScene'; +import { GrafanaTheme2 } from '@grafana/data'; +import { css } from '@emotion/css'; +import { BreakdownViewDefinition, breakdownViewsDefinitions } from './BreakdownViews'; + +export interface LogsActionBarSceneState extends SceneObjectState {} + +export class LogsActionBarScene extends SceneObjectBase { + public static Component = ({ model }: SceneComponentProps) => { + const styles = useStyles2(getStyles); + const exploration = getExplorationFor(model); + let currentBreakdownViewSlug = getDrilldownSlug(); + let allowNavToParent = false; + + if (!Object.values(PageSlugs).includes(currentBreakdownViewSlug)) { + const drilldownValueSlug = getDrilldownValueSlug(); + allowNavToParent = true; + if (drilldownValueSlug === ValueSlugs.field) { + currentBreakdownViewSlug = PageSlugs.fields; + } + if (drilldownValueSlug === ValueSlugs.label) { + currentBreakdownViewSlug = PageSlugs.labels; + } + } + + const serviceScene = sceneGraph.getAncestor(model, ServiceScene); + const { loading, $data, ...state } = serviceScene.useState(); + + return ( + +
+ + + +
+ + + {breakdownViewsDefinitions.map((tab, index) => { + return ( + { + if (tab.value !== currentBreakdownViewSlug || allowNavToParent) { + reportAppInteraction( + USER_EVENTS_PAGES.service_details, + USER_EVENTS_ACTIONS.service_details.action_view_changed, + { + newActionView: tab.value, + previousActionView: currentBreakdownViewSlug, + } + ); + if (tab.value) { + const serviceScene = sceneGraph.getAncestor(model, ServiceScene); + const variable = getLabelsVariable(serviceScene); + const service = variable.state.filters.find((f) => f.key === SERVICE_NAME); + + if (service?.value) { + navigateToDrilldownPage(tab.value, serviceScene); + } else { + navigateToIndex(); + } + } + } + }} + /> + ); + })} + + + ); + }; +} +const getCounter = (tab: BreakdownViewDefinition, state: ServiceSceneState) => { + switch (tab.value) { + case 'fields': + return state.fieldsCount ?? (state.fields?.filter((l) => l !== ALL_VARIABLE_VALUE) ?? []).length; + case 'patterns': + return state.patternsCount; + case 'labels': + return (state.labels?.filter((l) => l.label !== ALL_VARIABLE_VALUE) ?? []).length; + default: + return undefined; + } +}; + +function getStyles(theme: GrafanaTheme2) { + return { + actions: css({ + [theme.breakpoints.up(theme.breakpoints.values.md)]: { + position: 'absolute', + right: 0, + zIndex: 2, + }, + }), + }; +} diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index 177c539f2..34f6e966f 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -1,27 +1,25 @@ -import { css } from '@emotion/css'; import React from 'react'; -import { GrafanaTheme2, PanelData } from '@grafana/data'; +import { PanelData } from '@grafana/data'; import { SceneComponentProps, + SceneDataProvider, SceneFlexItem, SceneFlexLayout, sceneGraph, - SceneObject, SceneObjectBase, SceneObjectState, + SceneQueryRunner, SceneVariable, VariableDependencyConfig, } from '@grafana/scenes'; -import { Box, LoadingPlaceholder, Stack, Tab, TabsBar, useStyles2 } from '@grafana/ui'; -import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; +import { LoadingPlaceholder } from '@grafana/ui'; import { DetectedLabel, DetectedLabelsResponse, updateParserFromDataFrame } from 'services/fields'; import { getQueryRunner } from 'services/panel'; import { buildDataQuery, buildResourceQuery } from 'services/query'; -import { getDrilldownSlug, getDrilldownValueSlug, PageSlugs, PLUGIN_ID, ValueSlugs } from 'services/routing'; -import { getExplorationFor, getLokiDatasource } from 'services/scenes'; +import { getDrilldownSlug, getDrilldownValueSlug, PageSlugs, PLUGIN_ID } from 'services/routing'; +import { getLokiDatasource } from 'services/scenes'; import { - ALL_VARIABLE_VALUE, getLabelsVariable, LEVEL_VARIABLE_VALUE, LOG_STREAM_SELECTOR_EXPR, @@ -32,41 +30,23 @@ import { VAR_LEVELS, VAR_PATTERNS, } from 'services/variables'; -import { - buildFieldsBreakdownActionScene, - buildFieldValuesBreakdownActionScene, -} from './Breakdowns/FieldsBreakdownScene'; -import { buildLabelBreakdownActionScene, buildLabelValuesBreakdownActionScene } from './Breakdowns/LabelBreakdownScene'; -import { buildPatternsScene } from './Breakdowns/Patterns/PatternsBreakdownScene'; -import { GoToExploreButton } from './GoToExploreButton'; -import { buildLogsListScene } from './LogsListScene'; -import { testIds } from 'services/testIds'; import { sortLabelsByCardinality } from 'services/filters'; import { SERVICE_NAME } from 'Components/ServiceSelectionScene/ServiceSelectionScene'; import { getMetadataService } from '../../services/metadata'; import { navigateToDrilldownPage, navigateToIndex } from '../../services/navigate'; import { areArraysEqual } from '../../services/comparison'; +import { LogsActionBarScene } from './LogsActionBarScene'; +import { breakdownViewsDefinitions, valueBreakdownViews } from './BreakdownViews'; -interface BreakdownViewDefinition { - displayName: string; - value: PageSlugs; - testId: string; - getScene: (changeFields: (f: string[]) => void) => SceneObject; -} - -interface ValueBreakdownViewDefinition { - displayName: string; - value: ValueSlugs; - testId: string; - getScene: (value: string) => SceneObject; -} +const LOGS_PANEL_QUERY_REFID = 'logsPanelQuery'; +const PATTERNS_QUERY_REFID = 'patterns'; type MakeOptional = Pick, K> & Omit; export interface ServiceSceneCustomState { fields?: string[]; labels?: DetectedLabel[]; - // patterns?: LokiPattern[]; + patternsCount?: number; fieldsCount?: number; loading?: boolean; } @@ -74,16 +54,7 @@ export interface ServiceSceneCustomState { export interface ServiceSceneState extends SceneObjectState, ServiceSceneCustomState { body: SceneFlexLayout | undefined; drillDownLabel?: string; -} - -const LOGS_PANEL_QUERY_REFID = 'logsPanelQuery'; -const PATTERNS_QUERY_REFID = 'patterns'; - -function getServiceSceneQueryRunner() { - return getQueryRunner([ - buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID }), - buildResourceQuery(VAR_LABELS_EXPR, 'patterns', { refId: PATTERNS_QUERY_REFID }), - ]); + $data: SceneDataProvider; } export function getLogsPanelFrame(data: PanelData | undefined) { @@ -100,8 +71,7 @@ export class ServiceScene extends SceneObjectBase { onReferencedVariableValueChanged: this.onReferencedVariableValueChanged.bind(this), }); - public constructor(state: MakeOptional) { - console.log('service scene constructor'); + public constructor(state: MakeOptional) { super({ body: state.body ?? buildGraphScene(), $data: getServiceSceneQueryRunner(), @@ -164,39 +134,50 @@ export class ServiceScene extends SceneObjectBase { this.setBreakdownView(); this.setEmptyFiltersRedirection(); - if (this.state.$data) { - this._subs.add( - this.state.$data?.subscribeToState((newState) => { - // console.log('data change', newState) - const logsPanelResponse = getLogsPanelFrame(newState.data); - const patternsResponse = getPatternsFrames(newState.data); - // console.log('$data change', newState) - if (logsPanelResponse) { - this.updateFields(); - } else if (patternsResponse) { - console.log('patterns response', patternsResponse); - - // @todo fix - // this.setState({ - // patterns: patternsResponse.fields.map(pattern => { - // return { - // pattern: pattern.name, - // samples: pattern.values - // } - // }) - // }) - } - }) - ); - } + this._subs.add( + this.state.$data.subscribeToState((newState) => { + const logsPanelResponse = getLogsPanelFrame(newState.data); + const patternsResponse = getPatternsFrames(newState.data); + if (logsPanelResponse) { + this.updateFields(); + } + + if (patternsResponse?.length) { + // Save the count of patterns to state + this.setState({ + patternsCount: patternsResponse.length, + }); + } + }) + ); this.updateLabels(); - // this.updatePatterns(); + const labels = getLabelsVariable(this); + this._subs.add( + labels.subscribeToState((newState, prevState) => { + if (!areArraysEqual(newState.filters, prevState.filters)) { + const queryRunner = getQueryRunnerFromProvider(this.state.$data); + const newQueryRunner = getQueryRunnerFromProvider(getServiceSceneQueryRunner(true)); + + // If the queries changed, update the data provider + if (!areArraysEqual(queryRunner.state.queries, newQueryRunner.state.queries)) { + this.setState({ + $data: newQueryRunner, + }); + } + } + }) + ); + + // Update query runner on manual time range change this._subs.add( sceneGraph.getTimeRange(this).subscribeToState(() => { + this.setState({ + $data: getServiceSceneQueryRunner(true), + }); + this.updateLabels(); - // this.updatePatterns(); }) ); } @@ -261,7 +242,6 @@ export class ServiceScene extends SceneObjectBase { if (frame) { const res = updateParserFromDataFrame(frame, this); const fields = res.fields.filter((f) => !disabledFields.includes(f)).sort((a, b) => a.localeCompare(b)); - // console.log('fields', fields) if (!areArraysEqual(fields, this.state.fields)) { this.setState({ fields: fields, @@ -269,7 +249,6 @@ export class ServiceScene extends SceneObjectBase { }); } } else { - // console.log('clearing fields', newState.data) this.setState({ fields: [], loading: false, @@ -277,43 +256,6 @@ export class ServiceScene extends SceneObjectBase { } } - // private async updatePatterns() { - // const ds = await getLokiDatasource(this); - // if (!ds) { - // return; - // } - // - // const timeRange = sceneGraph.getTimeRange(this).state.value; - // const labels = getLabelsVariable(this); - // const fields = getFieldsVariable(this); - // - // const excludeLabels = [ALL_VARIABLE_VALUE, LEVEL_VARIABLE_VALUE]; - // - // const { data } = await ds.getResource( - // 'patterns', - // { - // query: renderLogQLStreamSelector([ - // // this will only be the service name for now - // ...labels.state.filters, - // // only include fields that are an indexed label - // ...fields.state.filters.filter( - // // we manually add level as a label, but it'll be structured metadata mostly, so we skip it here - // (field) => - // this.state.labels?.find((label) => label.label === field.key) && !excludeLabels.includes(field.key) - // ), - // ]), - // start: timeRange.from.utc().toISOString(), - // end: timeRange.to.utc().toISOString(), - // }, - // { - // headers: { - // 'X-Query-Tags': `Source=${PLUGIN_ID}`, - // }, - // } - // ); - // this.setState({ patterns: data }); - // } - private async updateLabels() { const ds = await getLokiDatasource(this); @@ -396,154 +338,42 @@ export class ServiceScene extends SceneObjectBase { }; } -const breakdownViewsDefinitions: BreakdownViewDefinition[] = [ - { - displayName: 'Logs', - value: PageSlugs.logs, - getScene: () => buildLogsListScene(), - testId: testIds.exploreServiceDetails.tabLogs, - }, - { - displayName: 'Labels', - value: PageSlugs.labels, - getScene: () => buildLabelBreakdownActionScene(), - testId: testIds.exploreServiceDetails.tabLabels, - }, - { - displayName: 'Fields', - value: PageSlugs.fields, - getScene: (f) => buildFieldsBreakdownActionScene(f), - testId: testIds.exploreServiceDetails.tabFields, - }, - { - displayName: 'Patterns', - value: PageSlugs.patterns, - getScene: () => buildPatternsScene(), - testId: testIds.exploreServiceDetails.tabPatterns, - }, -]; - -const valueBreakdownViews: ValueBreakdownViewDefinition[] = [ - { - displayName: 'Label', - value: ValueSlugs.label, - getScene: (value: string) => buildLabelValuesBreakdownActionScene(value), - testId: testIds.exploreServiceDetails.tabLabels, - }, - { - displayName: 'Field', - value: ValueSlugs.field, - getScene: (value: string) => buildFieldValuesBreakdownActionScene(value), - testId: testIds.exploreServiceDetails.tabFields, - }, -]; - -export interface LogsActionBarState extends SceneObjectState {} - -export class LogsActionBar extends SceneObjectBase { - public static Component = ({ model }: SceneComponentProps) => { - const styles = useStyles2(getStyles); - const exploration = getExplorationFor(model); - let currentBreakdownViewSlug = getDrilldownSlug(); - let allowNavToParent = false; - - if (!Object.values(PageSlugs).includes(currentBreakdownViewSlug)) { - const drilldownValueSlug = getDrilldownValueSlug(); - allowNavToParent = true; - if (drilldownValueSlug === ValueSlugs.field) { - currentBreakdownViewSlug = PageSlugs.fields; - } - if (drilldownValueSlug === ValueSlugs.label) { - currentBreakdownViewSlug = PageSlugs.labels; - } - } - - const serviceScene = sceneGraph.getAncestor(model, ServiceScene); - const { loading, $data, ...state } = serviceScene.useState(); - - const getCounter = (tab: BreakdownViewDefinition, state: ServiceSceneState) => { - switch (tab.value) { - case 'fields': - return state.fieldsCount ?? (state.fields?.filter((l) => l !== ALL_VARIABLE_VALUE) ?? []).length; - case 'patterns': - return getPatternsFrames($data?.state.data)?.length ?? 0; - case 'labels': - return (state.labels?.filter((l) => l.label !== ALL_VARIABLE_VALUE) ?? []).length; - default: - return undefined; - } - }; - - return ( - -
- - - -
- - - {breakdownViewsDefinitions.map((tab, index) => { - return ( - { - if (tab.value !== currentBreakdownViewSlug || allowNavToParent) { - reportAppInteraction( - USER_EVENTS_PAGES.service_details, - USER_EVENTS_ACTIONS.service_details.action_view_changed, - { - newActionView: tab.value, - previousActionView: currentBreakdownViewSlug, - } - ); - if (tab.value) { - const serviceScene = sceneGraph.getAncestor(model, ServiceScene); - const variable = getLabelsVariable(serviceScene); - const service = variable.state.filters.find((f) => f.key === SERVICE_NAME); - - if (service?.value) { - navigateToDrilldownPage(tab.value, serviceScene); - } else { - navigateToIndex(); - } - } - } - }} - /> - ); - })} - - - ); - }; -} - -function getStyles(theme: GrafanaTheme2) { - return { - actions: css({ - [theme.breakpoints.up(theme.breakpoints.values.md)]: { - position: 'absolute', - right: 0, - zIndex: 2, - }, - }), - }; -} - function buildGraphScene() { return new SceneFlexLayout({ direction: 'column', children: [ new SceneFlexItem({ ySizing: 'content', - body: new LogsActionBar({}), + body: new LogsActionBarScene({}), }), ], }); } + +function getServiceSceneQueryRunner(forceRefresh = false) { + const slug = getDrilldownSlug(); + const metadataService = getMetadataService(); + const state = metadataService.getServiceSceneState(); + + // We only need to query patterns on pages besides the patterns view to show the number of patterns in the tab. If that's already been set, let's skip requesting it again. + if (slug !== PageSlugs.patterns && state?.patternsCount !== undefined && !forceRefresh) { + return getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID })]); + } + + return getQueryRunner([ + buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID }), + buildResourceQuery(VAR_LABELS_EXPR, 'patterns', { refId: PATTERNS_QUERY_REFID }), + ]); +} + +function getQueryRunnerFromProvider(queryRunner: SceneDataProvider): SceneQueryRunner { + if (queryRunner instanceof SceneQueryRunner) { + return queryRunner; + } + + if (queryRunner.state.$data instanceof SceneQueryRunner) { + return queryRunner.state.$data; + } + + throw new Error('Cannot find query runner'); +} diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 3548e34f8..df2883c9c 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -73,21 +73,28 @@ class WrappedLokiDatasource extends RuntimeDataSource { throw new Error('Invalid datasource!'); } - request.targets.forEach((target) => { + const dataQueryRequest = { ...request }; + // override the target datasource to Loki + dataQueryRequest.targets = request.targets.map((target) => { + target.datasource = ds; + return target; + }); + + dataQueryRequest.targets.forEach((target) => { const requestType = target?.resource; let newSubscriber: Subscriber; switch (requestType) { case 'volume': { - newSubscriber = this.getVolume(request, ds, subscriber); + newSubscriber = this.getVolume(dataQueryRequest, ds, subscriber); break; } case 'patterns': { - newSubscriber = this.getPatterns(request, ds, subscriber); + newSubscriber = this.getPatterns(dataQueryRequest, ds, subscriber); break; } default: { - newSubscriber = this.getData(request, ds, subscriber); + newSubscriber = this.getData(dataQueryRequest, ds, subscriber); break; } } @@ -122,14 +129,9 @@ class WrappedLokiDatasource extends RuntimeDataSource { ) { const dataQueryRequest = { ...request }; // override the target datasource to Loki - dataQueryRequest.targets = request.targets - .filter((target) => { - return !target.resource; - }) - .map((target) => { - target.datasource = ds; - return target; - }); + dataQueryRequest.targets = request.targets.filter((target) => { + return !target.resource; + }); // query the datasource and return either observable or promise const dsResponse = ds.query(dataQueryRequest); diff --git a/src/services/metadata.ts b/src/services/metadata.ts index 708b81ec9..933d38464 100644 --- a/src/services/metadata.ts +++ b/src/services/metadata.ts @@ -21,7 +21,7 @@ export class MetadataService { this.serviceSceneState = { fields: state.fields, labels: state.labels, - // patterns: state.patterns, + patternsCount: state.patternsCount, fieldsCount: state.fieldsCount, loading: state.loading, }; From b786d67727981b38e536ce43fc0719836aa75181 Mon Sep 17 00:00:00 2001 From: Galen Date: Wed, 7 Aug 2024 11:31:51 -0500 Subject: [PATCH 04/37] chore: clean up --- .../ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx | 2 +- .../ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index b7e355f4d..09b421e7a 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -181,7 +181,7 @@ export class PatternsBreakdownScene extends SceneObjectBase { constructor(state?: Partial) { super({ - loading: false, + loading: true, ...state, legendSyncPatterns: new Set(), }); From 468361b8a07f2b49c7427564398cf15add5cedc3 Mon Sep 17 00:00:00 2001 From: Galen Date: Wed, 7 Aug 2024 11:36:17 -0500 Subject: [PATCH 05/37] chore: clean up --- project-words.txt | 1 + src/services/datasource.ts | 2 -- src/services/navigate.test.ts | 10 +--------- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/project-words.txt b/project-words.txt index 367e9275b..342c7dcea 100644 --- a/project-words.txt +++ b/project-words.txt @@ -469,3 +469,4 @@ subqueries dbscan svgs Dont +REFID diff --git a/src/services/datasource.ts b/src/services/datasource.ts index df2883c9c..5e93bd233 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -110,8 +110,6 @@ class WrappedLokiDatasource extends RuntimeDataSource { tap((response) => { if (response.state === LoadingState.Done || response.state === LoadingState.Error) { numberOfQueriesThatResolved++; - // @todo errors and cancelled queries (switch tabs while loading) - // @todo useDatasourcesFromTargets error in logs panel breaking context if (numberOfQueriesThatResolved < numberOfQueries) { response.state = LoadingState.Loading; } diff --git a/src/services/navigate.test.ts b/src/services/navigate.test.ts index 9457d55de..fdc896887 100644 --- a/src/services/navigate.test.ts +++ b/src/services/navigate.test.ts @@ -55,15 +55,7 @@ describe('navigate', () => { mockServiceSceneState = { labels, - // patterns: [ - // { - // pattern: 'error <_> message', - // samples: [ - // [1721220640, '270'], - // [1721220650, '341'], - // ], - // }, - // ], + patternsCount: 2, fields: ['field1', 'field2'], fieldsCount: 2, loading: true, From 6bda2fdb4aca57fa48b2ed7f261d2b542aa16006 Mon Sep 17 00:00:00 2001 From: Galen Date: Wed, 7 Aug 2024 12:14:03 -0500 Subject: [PATCH 06/37] test: fix test --- tests/exploreServicesBreakDown.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exploreServicesBreakDown.spec.ts b/tests/exploreServicesBreakDown.spec.ts index ac717b7c3..cdcb87578 100644 --- a/tests/exploreServicesBreakDown.spec.ts +++ b/tests/exploreServicesBreakDown.spec.ts @@ -6,6 +6,7 @@ import {mockEmptyQueryApiResponse} from "./mocks/mockEmptyQueryApiResponse"; import {LokiQuery} from "../src/services/query"; import {DataQueryRequest} from "@grafana/data"; import {DataQuery} from "@grafana/schema"; +import {LOGS_PANEL_QUERY_REFID} from "../src/Components/ServiceScene/ServiceScene"; test.describe('explore services breakdown page', () => { let explorePage: ExplorePage; @@ -113,7 +114,7 @@ test.describe('explore services breakdown page', () => { const postData = JSON.parse(rawPostData); const refId = postData.queries[0].refId // Field subqueries have a refId of the field name - if(refId !== 'A'){ + if(refId !== 'logsPanelQuery' && refId !== 'A'){ requestCount++ return await route.fulfill({json: mockResponse}) } From a34dc57280719d83d505872805cfbc117e44e977 Mon Sep 17 00:00:00 2001 From: Galen Date: Thu, 8 Aug 2024 13:28:45 -0500 Subject: [PATCH 07/37] fix: refactor and fix buggy re-rendering in logs panels --- src/Components/ServiceScene/LogsListScene.tsx | 108 +++-------- .../ServiceScene/LogsPanelScene.tsx | 173 ++++++++++++++++++ .../ServiceScene/LogsTableScene.tsx | 53 +++++- 3 files changed, 238 insertions(+), 96 deletions(-) create mode 100644 src/Components/ServiceScene/LogsPanelScene.tsx diff --git a/src/Components/ServiceScene/LogsListScene.tsx b/src/Components/ServiceScene/LogsListScene.tsx index 66083118d..f88d9518c 100644 --- a/src/Components/ServiceScene/LogsListScene.tsx +++ b/src/Components/ServiceScene/LogsListScene.tsx @@ -1,11 +1,11 @@ import React from 'react'; import { - AdHocFiltersVariable, - PanelBuilders, SceneComponentProps, + SceneDataNode, SceneFlexItem, SceneFlexLayout, + sceneGraph, SceneObjectBase, SceneObjectState, SceneObjectUrlSyncConfig, @@ -15,17 +15,13 @@ import { import { LineFilterScene } from './LineFilterScene'; import { SelectedTableRow } from '../Table/LogLineCellComponent'; import { LogsTableScene } from './LogsTableScene'; -import { LogsPanelHeaderActions } from '../Table/LogsHeaderActions'; import { LogsVolumePanel } from './LogsVolumePanel'; import { css } from '@emotion/css'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '../../services/analytics'; -import { DataFrame } from '@grafana/data'; -import { addToFilters, FilterType } from './Breakdowns/AddToFiltersButton'; -import { getLabelTypeFromFrame, LabelType } from 'services/fields'; -import { getAdHocFiltersVariable, VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from 'services/variables'; import { locationService } from '@grafana/runtime'; import { LogOptionsScene } from './LogOptionsScene'; -import { getLogOption } from 'services/store'; +import { LogsPanelScene } from './LogsPanelScene'; +import { LoadingState } from '@grafana/data'; export interface LogsListSceneState extends SceneObjectState { loading?: boolean; @@ -115,6 +111,10 @@ export class LogsListScene extends SceneObjectBase { ); } + public getLineFilterScene() { + return this.lineFilterScene; + } + private setStateFromUrl(searchParams: URLSearchParams) { const state: Partial = {}; const selectedLineUrl = searchParams.get('selectedLine'); @@ -137,90 +137,12 @@ export class LogsListScene extends SceneObjectBase { } } - private handleLabelFilter(key: string, value: string, frame: DataFrame | undefined, operator: FilterType) { - // @TODO: NOOP. We need a way to let the user know why this is not possible. - if (key === 'service_name') { - return; - } - const type = frame ? getLabelTypeFromFrame(key, frame) : LabelType.Parsed; - const variableName = type === LabelType.Indexed ? VAR_LABELS : VAR_FIELDS; - addToFilters(key, value, operator, this, variableName); - - reportAppInteraction( - USER_EVENTS_PAGES.service_details, - USER_EVENTS_ACTIONS.service_details.logs_detail_filter_applied, - { - filterType: variableName, - key, - action: operator, - } - ); - } - - public handleLabelFilterClick = (key: string, value: string, frame?: DataFrame) => { - this.handleLabelFilter(key, value, frame, 'toggle'); - }; - - public handleLabelFilterOutClick = (key: string, value: string, frame?: DataFrame) => { - this.handleLabelFilter(key, value, frame, 'exclude'); - }; - - public handleIsFilterLabelActive = (key: string, value: string) => { - const labels = getAdHocFiltersVariable(VAR_LABELS, this); - const fields = getAdHocFiltersVariable(VAR_FIELDS, this); - const levels = getAdHocFiltersVariable(VAR_LEVELS, this); - - const hasKeyValueFilter = (filter: AdHocFiltersVariable | null) => - filter && - filter.state.filters.findIndex( - (filter) => filter.operator === '=' && filter.key === key && filter.value === value - ) >= 0; - - return hasKeyValueFilter(labels) || hasKeyValueFilter(fields) || hasKeyValueFilter(levels); - }; - - public handleFilterStringClick = (value: string) => { - if (this.lineFilterScene) { - this.lineFilterScene.updateFilter(value); - reportAppInteraction( - USER_EVENTS_PAGES.service_details, - USER_EVENTS_ACTIONS.service_details.logs_popover_line_filter, - { - selectionLength: value.length, - } - ); - } - }; - public updateLogsPanel = () => { this.setState({ panel: this.getVizPanel(), }); }; - private getLogsPanel() { - const visualizationType = this.state.visualizationType; - - return new SceneFlexItem({ - height: 'calc(100vh - 220px)', - body: PanelBuilders.logs() - .setTitle('Logs') - .setOption('showTime', true) - // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. - .setOption('onClickFilterLabel', this.handleLabelFilterClick) - // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. - .setOption('onClickFilterOutLabel', this.handleLabelFilterOutClick) - // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. - .setOption('isFilterLabelActive', this.handleIsFilterLabelActive) - // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. - .setOption('onClickFilterString', this.handleFilterStringClick) - .setOption('wrapLogMessage', Boolean(getLogOption('wrapLines'))) - .setOption('showLogContextToggle', true) - .setHeaderActions() - .build(), - }); - } - public setVisualizationType = (type: LogsVisualizationType) => { this.setState({ visualizationType: type, @@ -252,7 +174,19 @@ export class LogsListScene extends SceneObjectBase { new LogOptionsScene(), ], }), - this.getLogsPanel(), + new SceneFlexItem({ + height: 'calc(100vh - 220px)', + body: new LogsPanelScene({ + // this data node is just for the initial loading state before we get our first response back + data: new SceneDataNode({ + data: { + state: LoadingState.Loading, + series: [], + timeRange: sceneGraph.getTimeRange(this).state.value, + }, + }).state, + }), + }), ] : [ new SceneFlexItem({ diff --git a/src/Components/ServiceScene/LogsPanelScene.tsx b/src/Components/ServiceScene/LogsPanelScene.tsx new file mode 100644 index 000000000..f9ad1e5a7 --- /dev/null +++ b/src/Components/ServiceScene/LogsPanelScene.tsx @@ -0,0 +1,173 @@ +import { + AdHocFiltersVariable, + PanelBuilders, + SceneComponentProps, + SceneDataNode, + SceneDataState, + sceneGraph, + SceneObjectBase, + SceneObjectState, + VizPanel, +} from '@grafana/scenes'; +import { DataFrame, LoadingState } from '@grafana/data'; +import { getLogsPanelFrame } from './ServiceScene'; +import { getLogOption } from '../../services/store'; +import { LogsPanelHeaderActions } from '../Table/LogsHeaderActions'; +import React from 'react'; +import { LogsListScene } from './LogsListScene'; +import { LoadingPlaceholder } from '@grafana/ui'; +import { addToFilters, FilterType } from './Breakdowns/AddToFiltersButton'; +import { getLabelTypeFromFrame, LabelType } from '../../services/fields'; +import { getAdHocFiltersVariable, VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from '../../services/variables'; +import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '../../services/analytics'; +import { areArraysEqual } from '../../services/comparison'; + +interface LogsPanelSceneState extends SceneObjectState { + data: SceneDataState; + body?: VizPanel; +} + +export class LogsPanelScene extends SceneObjectBase { + constructor(state: Partial & { data: SceneDataState }) { + super({ + ...state, + }); + + this.addActivationHandler(this.onActivate.bind(this)); + } + + public onActivate() { + if (!this.state.body) { + this.setState({ + body: this.getLogsPanel(), + }); + } + + this._subs.add( + sceneGraph.getData(this).subscribeToState((newState, prevState) => { + if (!areArraysEqual(newState.data?.series, prevState.data?.series)) { + const dataFrame = getLogsPanelFrame(newState.data); + + // If we have a response, set it + if (dataFrame && newState.data) { + this.setState({ + data: newState, + }); + + // And update data node loading state + this.state.body?.state.$data?.setState({ + data: { + ...newState.data, + state: LoadingState.Done, + }, + }); + } else if (this.state.body?.state.$data && this.state.body?.state.$data.state.data) { + // otherwise set a loading state in the viz + this.state.body.state.$data.setState({ + ...this.state.body.state.$data.state, + data: { + ...this.state.body.state.$data.state.data, + state: LoadingState.Loading, + }, + }); + } + } + }) + ); + } + + private getLogsPanel() { + const parentModel = sceneGraph.getAncestor(this, LogsListScene); + const visualizationType = parentModel.state.visualizationType; + + const dataNode = new SceneDataNode({ + data: this.state.data?.data, + }); + + return ( + PanelBuilders.logs() + .setTitle('Logs') + .setData(dataNode) + .setOption('showTime', true) + // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. + .setOption('onClickFilterLabel', this.handleLabelFilterClick) + // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. + .setOption('onClickFilterOutLabel', this.handleLabelFilterOutClick) + // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. + .setOption('isFilterLabelActive', this.handleIsFilterLabelActive) + // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. + .setOption('onClickFilterString', this.handleFilterStringClick) + .setOption('wrapLogMessage', Boolean(getLogOption('wrapLines'))) + .setOption('showLogContextToggle', true) + .setHeaderActions( + + ) + .build() + ); + } + + private handleLabelFilterClick = (key: string, value: string, frame?: DataFrame) => { + this.handleLabelFilter(key, value, frame, 'toggle'); + }; + + private handleLabelFilterOutClick = (key: string, value: string, frame?: DataFrame) => { + this.handleLabelFilter(key, value, frame, 'exclude'); + }; + + private handleIsFilterLabelActive = (key: string, value: string) => { + const labels = getAdHocFiltersVariable(VAR_LABELS, this); + const fields = getAdHocFiltersVariable(VAR_FIELDS, this); + const levels = getAdHocFiltersVariable(VAR_LEVELS, this); + + const hasKeyValueFilter = (filter: AdHocFiltersVariable | null) => + filter && + filter.state.filters.findIndex( + (filter) => filter.operator === '=' && filter.key === key && filter.value === value + ) >= 0; + + return hasKeyValueFilter(labels) || hasKeyValueFilter(fields) || hasKeyValueFilter(levels); + }; + + private handleFilterStringClick = (value: string) => { + const parentModel = sceneGraph.getAncestor(this, LogsListScene); + const lineFilterScene = parentModel.getLineFilterScene(); + if (lineFilterScene) { + lineFilterScene.updateFilter(value); + reportAppInteraction( + USER_EVENTS_PAGES.service_details, + USER_EVENTS_ACTIONS.service_details.logs_popover_line_filter, + { + selectionLength: value.length, + } + ); + } + }; + + private handleLabelFilter(key: string, value: string, frame: DataFrame | undefined, operator: FilterType) { + // @TODO: NOOP. We need a way to let the user know why this is not possible. + if (key === 'service_name') { + return; + } + const type = frame ? getLabelTypeFromFrame(key, frame) : LabelType.Parsed; + const variableName = type === LabelType.Indexed ? VAR_LABELS : VAR_FIELDS; + addToFilters(key, value, operator, this, variableName); + + reportAppInteraction( + USER_EVENTS_PAGES.service_details, + USER_EVENTS_ACTIONS.service_details.logs_detail_filter_applied, + { + filterType: variableName, + key, + action: operator, + } + ); + } + + public static Component = ({ model }: SceneComponentProps) => { + const { body } = model.useState(); + if (body) { + return ; + } + return ; + }; +} diff --git a/src/Components/ServiceScene/LogsTableScene.tsx b/src/Components/ServiceScene/LogsTableScene.tsx index b3f8f4432..3e718d4a0 100644 --- a/src/Components/ServiceScene/LogsTableScene.tsx +++ b/src/Components/ServiceScene/LogsTableScene.tsx @@ -1,6 +1,6 @@ -import { SceneComponentProps, sceneGraph, SceneObjectBase } from '@grafana/scenes'; +import { SceneComponentProps, SceneDataState, sceneGraph, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; import { LogsListScene } from './LogsListScene'; -import { AdHocVariableFilter } from '@grafana/data'; +import { AdHocVariableFilter, LoadingState } from '@grafana/data'; import { TableProvider } from '../Table/TableProvider'; import React, { useRef } from 'react'; import { PanelChrome } from '@grafana/ui'; @@ -10,14 +10,50 @@ import { addAdHocFilter } from './Breakdowns/AddToFiltersButton'; import { areArraysEqual } from '../../services/comparison'; import { getLogsPanelFrame } from './ServiceScene'; -export class LogsTableScene extends SceneObjectBase { +interface LogsTableSceneState extends SceneObjectState { + data?: SceneDataState; + loading?: LoadingState; +} + +export class LogsTableScene extends SceneObjectBase { + constructor(state: Partial) { + super(state); + + this.addActivationHandler(this.onActivate.bind(this)); + } + + /** + * We can't subscribe to the state of the data provider anymore, because there are multiple queries running in each data provider + * So we need to manually update the data state to prevent unnecessary re-renders that cause flickering and break loading states + */ + public onActivate() { + this._subs.add( + sceneGraph.getData(this).subscribeToState((newState, prevState) => { + const dataFrame = getLogsPanelFrame(newState.data); + + // Just this query is done + if (dataFrame) { + this.setState({ + data: newState, + loading: newState.data?.state, + }); + } else { + // Query is loading + this.setState({ + loading: newState.data?.state, + }); + } + }) + ); + } public static Component = ({ model }: SceneComponentProps) => { + const styles = getStyles(); // Get state from parent model const parentModel = sceneGraph.getAncestor(model, LogsListScene); const { selectedLine, urlColumns, visualizationType } = parentModel.useState(); - // Get dataFrame - const { data } = sceneGraph.getData(model).useState(); + // Get data state + const { data, loading } = model.useState(); // Get time range const timeRange = sceneGraph.getTimeRange(model); @@ -44,17 +80,16 @@ export class LogsTableScene extends SceneObjectBase { } }; - const styles = getStyles(); - const dataFrame = getLogsPanelFrame(data); + const dataFrame = getLogsPanelFrame(data?.data); return (
} > - {data?.series[0] && ( + {dataFrame && ( Date: Thu, 8 Aug 2024 13:46:27 -0500 Subject: [PATCH 08/37] fix: fix data node not being set from scene cache on route/activation --- src/Components/ServiceScene/LogsListScene.tsx | 24 ++++++++++++------- .../ServiceScene/LogsTableScene.tsx | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/Components/ServiceScene/LogsListScene.tsx b/src/Components/ServiceScene/LogsListScene.tsx index f88d9518c..fa37efa89 100644 --- a/src/Components/ServiceScene/LogsListScene.tsx +++ b/src/Components/ServiceScene/LogsListScene.tsx @@ -160,6 +160,18 @@ export class LogsListScene extends SceneObjectBase { private getVizPanel() { this.lineFilterScene = new LineFilterScene(); + + // If this is called before the query has executed, we need to return an empty data node to init the loading state, instead of no-data + const data = + sceneGraph.getData(this).state ?? + new SceneDataNode({ + data: { + state: LoadingState.Loading, + series: [], + timeRange: sceneGraph.getTimeRange(this).state.value, + }, + }).state; + return new SceneFlexLayout({ direction: 'column', children: @@ -178,13 +190,7 @@ export class LogsListScene extends SceneObjectBase { height: 'calc(100vh - 220px)', body: new LogsPanelScene({ // this data node is just for the initial loading state before we get our first response back - data: new SceneDataNode({ - data: { - state: LoadingState.Loading, - series: [], - timeRange: sceneGraph.getTimeRange(this).state.value, - }, - }).state, + data, }), }), ] @@ -195,7 +201,9 @@ export class LogsListScene extends SceneObjectBase { }), new SceneFlexItem({ height: 'calc(100vh - 220px)', - body: new LogsTableScene({}), + body: new LogsTableScene({ + data, + }), }), ], }); diff --git a/src/Components/ServiceScene/LogsTableScene.tsx b/src/Components/ServiceScene/LogsTableScene.tsx index 3e718d4a0..869e418ba 100644 --- a/src/Components/ServiceScene/LogsTableScene.tsx +++ b/src/Components/ServiceScene/LogsTableScene.tsx @@ -11,12 +11,12 @@ import { areArraysEqual } from '../../services/comparison'; import { getLogsPanelFrame } from './ServiceScene'; interface LogsTableSceneState extends SceneObjectState { - data?: SceneDataState; + data: SceneDataState; loading?: LoadingState; } export class LogsTableScene extends SceneObjectBase { - constructor(state: Partial) { + constructor(state: Partial & { data: SceneDataState }) { super(state); this.addActivationHandler(this.onActivate.bind(this)); From 1c2e57b9ec4671d6d827f2dea1206db97232cf20 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 13:27:22 -0500 Subject: [PATCH 09/37] fix: removing multiple queries per query runner, adding multiple query runners --- src/Components/IndexScene/IndexScene.tsx | 4 +- .../Patterns/PatternsBreakdownScene.tsx | 26 +---- .../Patterns/PatternsFrameScene.tsx | 10 +- src/Components/ServiceScene/ServiceScene.tsx | 108 +++++++++--------- src/services/LogsSceneQueryRunner.ts | 18 +++ src/services/datasource.ts | 1 + src/services/metadata.ts | 9 ++ src/services/panel.ts | 8 ++ 8 files changed, 105 insertions(+), 79 deletions(-) create mode 100644 src/services/LogsSceneQueryRunner.ts diff --git a/src/Components/IndexScene/IndexScene.tsx b/src/Components/IndexScene/IndexScene.tsx index 77fab17f6..6e871f70c 100644 --- a/src/Components/IndexScene/IndexScene.tsx +++ b/src/Components/IndexScene/IndexScene.tsx @@ -161,7 +161,9 @@ function getContentScene(drillDownLabel?: string) { return new ServiceSelectionScene({}); } - return new ServiceScene({ drillDownLabel }); + return new ServiceScene({ + drillDownLabel, + }); } function getVariableSet(initialDatasourceUid: string, initialFilters?: AdHocVariableFilter[]) { diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index 09b421e7a..d5b199a44 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -22,7 +22,6 @@ import { PatternsFrameScene } from './PatternsFrameScene'; import { PatternsViewTextSearch } from './PatternsViewTextSearch'; import { PatternsNotDetected, PatternsTooOld } from './PatternsNotDetected'; import { areArraysEqual } from '../../../../services/comparison'; -import { Unsubscribable } from 'rxjs'; export interface PatternsBreakdownSceneState extends SceneObjectState { body?: SceneFlexLayout; @@ -36,7 +35,6 @@ export interface PatternsBreakdownSceneState extends SceneObjectState { // Subset of patternFrames, undefined if empty, empty array if search results returned nothing (no data) filteredPatterns?: PatternFrame[]; patternFilter: string; - dataSub?: Unsubscribable; } export type PatternFrame = { @@ -69,8 +67,8 @@ export class PatternsBreakdownScene extends SceneObjectBase= PATTERNS_MAX_AGE_HOURS; @@ -105,7 +103,7 @@ export class PatternsBreakdownScene extends SceneObjectBase { - if (newState.$data.state.key !== prevState.$data.state.key) { - const dataSub = serviceScene.state.$data.subscribeToState(this.onDataProviderChange); - this.state.dataSub?.unsubscribe(); - this.setState({ - dataSub, - loading: true, - }); - } - }); + this._subs.add(serviceScene.state.$patternsData.subscribeToState(this.onDataProviderChange)); } private onDataProviderChange = (newState: SceneDataState, prevState: SceneDataState) => { diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx index d6181c086..cb97bbb64 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx @@ -44,8 +44,8 @@ export class PatternsFrameScene extends SceneObjectBase public static Component = ({ model }: SceneComponentProps) => { const { body, loading } = model.useState(); const logsByServiceScene = sceneGraph.getAncestor(model, ServiceScene); - const { $data } = logsByServiceScene.useState(); - const patterns = getPatternsFrames($data?.state.data); + const { $patternsData } = logsByServiceScene.useState(); + const patterns = getPatternsFrames($patternsData?.state.data); return (
@@ -60,8 +60,8 @@ export class PatternsFrameScene extends SceneObjectBase // If the patterns have changed, recalculate the dataframes this._subs.add( sceneGraph.getAncestor(this, ServiceScene).subscribeToState((newState, prevState) => { - const newFrame = getPatternsFrames(newState?.$data?.state?.data); - const prevFrame = getPatternsFrames(prevState?.$data?.state?.data); + const newFrame = getPatternsFrames(newState?.$patternsData?.state?.data); + const prevFrame = getPatternsFrames(prevState?.$patternsData?.state?.data); if (!areArraysEqual(newFrame, prevFrame)) { const patternsBreakdownScene = sceneGraph.getAncestor(this, PatternsBreakdownScene); @@ -113,7 +113,7 @@ export class PatternsFrameScene extends SceneObjectBase const serviceScene = sceneGraph.getAncestor(this, ServiceScene); - const lokiPatterns = getPatternsFrames(serviceScene.state.$data?.state.data); + const lokiPatterns = getPatternsFrames(serviceScene.state.$patternsData?.state.data); if (!lokiPatterns || !patternFrames) { console.warn('Failed to update PatternsFrameScene body'); return; diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index 34f6e966f..cd08b851e 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -1,6 +1,6 @@ import React from 'react'; -import { PanelData } from '@grafana/data'; +import { LoadingState, PanelData } from '@grafana/data'; import { SceneComponentProps, SceneDataProvider, @@ -15,7 +15,7 @@ import { } from '@grafana/scenes'; import { LoadingPlaceholder } from '@grafana/ui'; import { DetectedLabel, DetectedLabelsResponse, updateParserFromDataFrame } from 'services/fields'; -import { getQueryRunner } from 'services/panel'; +import { getQueryRunner, getResourceQueryRunner } from 'services/panel'; import { buildDataQuery, buildResourceQuery } from 'services/query'; import { getDrilldownSlug, getDrilldownValueSlug, PageSlugs, PLUGIN_ID } from 'services/routing'; import { getLokiDatasource } from 'services/scenes'; @@ -55,6 +55,7 @@ export interface ServiceSceneState extends SceneObjectState, ServiceSceneCustomS body: SceneFlexLayout | undefined; drillDownLabel?: string; $data: SceneDataProvider; + $patternsData: SceneQueryRunner; } export function getLogsPanelFrame(data: PanelData | undefined) { @@ -71,11 +72,12 @@ export class ServiceScene extends SceneObjectBase { onReferencedVariableValueChanged: this.onReferencedVariableValueChanged.bind(this), }); - public constructor(state: MakeOptional) { + public constructor(state: MakeOptional) { super({ + loading: true, body: state.body ?? buildGraphScene(), $data: getServiceSceneQueryRunner(), - loading: true, + $patternsData: getPatternsQueryRunner(), ...state, }); @@ -106,7 +108,11 @@ export class ServiceScene extends SceneObjectBase { this.setState({ $data: undefined, body: undefined, + $patternsData: undefined, + patternsCount: undefined, }); + getMetadataService().setServiceSceneState(this.state); + this._subs.unsubscribe(); // Redirect to root with updated params, which will trigger history push back to index route, preventing empty page or empty service query bugs navigateToIndex(); } @@ -133,20 +139,35 @@ export class ServiceScene extends SceneObjectBase { this.setBreakdownView(); this.setEmptyFiltersRedirection(); + const slug = getDrilldownSlug(); + + // If we don't have a patterns count in the tabs, or we are activating the patterns scene, run the pattern query + if (this.state.patternsCount === undefined || slug === 'patterns') { + this.state.$patternsData.runQueries(); + } this._subs.add( this.state.$data.subscribeToState((newState) => { - const logsPanelResponse = getLogsPanelFrame(newState.data); - const patternsResponse = getPatternsFrames(newState.data); - if (logsPanelResponse) { - this.updateFields(); + if (newState.data?.state === LoadingState.Done) { + const logsPanelResponse = getLogsPanelFrame(newState.data); + if (logsPanelResponse) { + this.updateFields(); + } } + }) + ); - if (patternsResponse?.length) { - // Save the count of patterns to state - this.setState({ - patternsCount: patternsResponse.length, - }); + this._subs.add( + this.state.$patternsData.subscribeToState((newState) => { + if (newState.data?.state === LoadingState.Done) { + const patternsResponse = getPatternsFrames(newState.data); + if (patternsResponse?.length !== undefined) { + // Save the count of patterns to state + this.setState({ + patternsCount: patternsResponse.length, + }); + getMetadataService().setPatternsCount(patternsResponse.length); + } } }) ); @@ -157,15 +178,7 @@ export class ServiceScene extends SceneObjectBase { this._subs.add( labels.subscribeToState((newState, prevState) => { if (!areArraysEqual(newState.filters, prevState.filters)) { - const queryRunner = getQueryRunnerFromProvider(this.state.$data); - const newQueryRunner = getQueryRunnerFromProvider(getServiceSceneQueryRunner(true)); - - // If the queries changed, update the data provider - if (!areArraysEqual(queryRunner.state.queries, newQueryRunner.state.queries)) { - this.setState({ - $data: newQueryRunner, - }); - } + this.state.$patternsData.runQueries(); } }) ); @@ -173,11 +186,8 @@ export class ServiceScene extends SceneObjectBase { // Update query runner on manual time range change this._subs.add( sceneGraph.getTimeRange(this).subscribeToState(() => { - this.setState({ - $data: getServiceSceneQueryRunner(true), - }); - this.updateLabels(); + this.state.$patternsData.runQueries(); }) ); } @@ -189,6 +199,10 @@ export class ServiceScene extends SceneObjectBase { stateUpdate.$data = getServiceSceneQueryRunner(); } + if (!this.state.$patternsData) { + stateUpdate.$patternsData = getPatternsQueryRunner(); + } + if (!this.state.body) { stateUpdate.body = buildGraphScene(); } @@ -263,7 +277,6 @@ export class ServiceScene extends SceneObjectBase { return; } const timeRange = sceneGraph.getTimeRange(this); - const timeRangeValue = timeRange.state.value; const filters = getLabelsVariable(this); @@ -350,30 +363,23 @@ function buildGraphScene() { }); } -function getServiceSceneQueryRunner(forceRefresh = false) { - const slug = getDrilldownSlug(); - const metadataService = getMetadataService(); - const state = metadataService.getServiceSceneState(); - - // We only need to query patterns on pages besides the patterns view to show the number of patterns in the tab. If that's already been set, let's skip requesting it again. - if (slug !== PageSlugs.patterns && state?.patternsCount !== undefined && !forceRefresh) { - return getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID })]); - } - - return getQueryRunner([ - buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID }), - buildResourceQuery(VAR_LABELS_EXPR, 'patterns', { refId: PATTERNS_QUERY_REFID }), - ]); +// @todo move to datasource or queries? +export function getPatternsQueryRunner() { + return getResourceQueryRunner([buildResourceQuery(VAR_LABELS_EXPR, 'patterns', { refId: PATTERNS_QUERY_REFID })]); } -function getQueryRunnerFromProvider(queryRunner: SceneDataProvider): SceneQueryRunner { - if (queryRunner instanceof SceneQueryRunner) { - return queryRunner; - } - - if (queryRunner.state.$data instanceof SceneQueryRunner) { - return queryRunner.state.$data; - } - - throw new Error('Cannot find query runner'); +function getServiceSceneQueryRunner() { + return getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID })]); } +// +// function getQueryRunnerFromProvider(queryRunner: SceneDataProvider): SceneQueryRunner { +// if (queryRunner instanceof SceneQueryRunner) { +// return queryRunner; +// } +// +// if (queryRunner.state.$data instanceof SceneQueryRunner) { +// return queryRunner.state.$data; +// } +// +// throw new Error('Cannot find query runner'); +// } diff --git a/src/services/LogsSceneQueryRunner.ts b/src/services/LogsSceneQueryRunner.ts new file mode 100644 index 000000000..647224e40 --- /dev/null +++ b/src/services/LogsSceneQueryRunner.ts @@ -0,0 +1,18 @@ +import { QueryRunnerState, sceneGraph, SceneQueryRunner } from '@grafana/scenes'; + +export class LogsSceneQueryRunner extends SceneQueryRunner { + constructor(initialState: QueryRunnerState) { + super(initialState); + } + + public runQueries() { + const timeRange = sceneGraph.getTimeRange(this); + + // We don't want to subscribe to time range changes, or we'll get duplicate queries + // this.subscribeToTimeRangeChanges(timeRange); + + // Hack to call private method + // @todo can we make runWithTimeRange protected? + this['runWithTimeRange'](timeRange); + } +} diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 5e93bd233..823752cfb 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -90,6 +90,7 @@ class WrappedLokiDatasource extends RuntimeDataSource { break; } case 'patterns': { + console.log('patterns called'); newSubscriber = this.getPatterns(dataQueryRequest, ds, subscriber); break; } diff --git a/src/services/metadata.ts b/src/services/metadata.ts index 933d38464..8a4339348 100644 --- a/src/services/metadata.ts +++ b/src/services/metadata.ts @@ -17,6 +17,15 @@ export class MetadataService { return this.serviceSceneState; } + public setPatternsCount(count: number) { + if (this.serviceSceneState) { + this.serviceSceneState.patternsCount = count; + } else { + this.serviceSceneState = {}; + this.serviceSceneState.patternsCount = count; + } + } + public setServiceSceneState(state: ServiceSceneCustomState) { this.serviceSceneState = { fields: state.fields, diff --git a/src/services/panel.ts b/src/services/panel.ts index 16a5faddb..f413842df 100644 --- a/src/services/panel.ts +++ b/src/services/panel.ts @@ -4,6 +4,7 @@ import { map, Observable } from 'rxjs'; import { LokiQuery } from './query'; import { HideSeriesConfig } from '@grafana/schema'; import { WRAPPED_LOKI_DS_UID } from './datasource'; +import { LogsSceneQueryRunner } from './LogsSceneQueryRunner'; const UNKNOWN_LEVEL_LOGS = 'logs'; export function setLeverColorOverrides(overrides: FieldConfigOverridesBuilder) { @@ -78,6 +79,13 @@ export function sortLevelTransformation() { }; } +export function getResourceQueryRunner(queries: LokiQuery[]) { + return new LogsSceneQueryRunner({ + datasource: { uid: WRAPPED_LOKI_DS_UID }, + queries: queries, + }); +} + export function getQueryRunner(queries: LokiQuery[]) { // if there's a legendFormat related to any `level` like label, we want to // sort the output equally. That's purposefully not `LEVEL_VARIABLE_VALUE`, From c3a6182bb9b0b8dafe89acad7b83355a2caf18b3 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 13:31:30 -0500 Subject: [PATCH 10/37] chore: remove console.log --- src/services/datasource.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 823752cfb..3a0d43e1f 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -82,27 +82,21 @@ class WrappedLokiDatasource extends RuntimeDataSource { dataQueryRequest.targets.forEach((target) => { const requestType = target?.resource; - let newSubscriber: Subscriber; switch (requestType) { case 'volume': { - newSubscriber = this.getVolume(dataQueryRequest, ds, subscriber); + this.getVolume(dataQueryRequest, ds, subscriber); break; } case 'patterns': { - console.log('patterns called'); - newSubscriber = this.getPatterns(dataQueryRequest, ds, subscriber); + this.getPatterns(dataQueryRequest, ds, subscriber); break; } default: { - newSubscriber = this.getData(dataQueryRequest, ds, subscriber); + this.getData(dataQueryRequest, ds, subscriber); break; } } - - if (newSubscriber) { - subscriber.add(newSubscriber); - } }); }); }); From 365a4ef9b4bde29dc4eca4948550b2a7a8a46388 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 13:46:19 -0500 Subject: [PATCH 11/37] chore: docs --- src/services/LogsSceneQueryRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/LogsSceneQueryRunner.ts b/src/services/LogsSceneQueryRunner.ts index 647224e40..a9a83c87f 100644 --- a/src/services/LogsSceneQueryRunner.ts +++ b/src/services/LogsSceneQueryRunner.ts @@ -11,8 +11,8 @@ export class LogsSceneQueryRunner extends SceneQueryRunner { // We don't want to subscribe to time range changes, or we'll get duplicate queries // this.subscribeToTimeRangeChanges(timeRange); + // @todo can we make runWithTimeRange protected? (https://github.com/grafana/scenes/pull/866) // Hack to call private method - // @todo can we make runWithTimeRange protected? this['runWithTimeRange'](timeRange); } } From 982d76cdfb3cbacbee6a1e8fbb95b1cddfef8441 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 14:39:12 -0500 Subject: [PATCH 12/37] chore: why are the videos gone in failed playwright reports? --- playwright.config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index c7dd8f133..e21e5ef0d 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -34,9 +34,9 @@ export default defineConfig({ trace: 'on-first-retry', // Turn on when debugging local tests - // video: { - // mode: 'on', - // } + video: { + mode: 'on-first-retry', + } }, expect: { timeout: 15000 }, From 1ef913a7c915f52c57484d26e26219e00779f24f Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 14:45:56 -0500 Subject: [PATCH 13/37] chore: clean up --- src/Components/ServiceScene/LogsListScene.tsx | 23 +--------- .../ServiceScene/LogsPanelScene.tsx | 46 +------------------ .../ServiceScene/LogsTableScene.tsx | 31 +++---------- 3 files changed, 10 insertions(+), 90 deletions(-) diff --git a/src/Components/ServiceScene/LogsListScene.tsx b/src/Components/ServiceScene/LogsListScene.tsx index fa37efa89..b1a6f312c 100644 --- a/src/Components/ServiceScene/LogsListScene.tsx +++ b/src/Components/ServiceScene/LogsListScene.tsx @@ -2,10 +2,8 @@ import React from 'react'; import { SceneComponentProps, - SceneDataNode, SceneFlexItem, SceneFlexLayout, - sceneGraph, SceneObjectBase, SceneObjectState, SceneObjectUrlSyncConfig, @@ -21,7 +19,6 @@ import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '.. import { locationService } from '@grafana/runtime'; import { LogOptionsScene } from './LogOptionsScene'; import { LogsPanelScene } from './LogsPanelScene'; -import { LoadingState } from '@grafana/data'; export interface LogsListSceneState extends SceneObjectState { loading?: boolean; @@ -161,17 +158,6 @@ export class LogsListScene extends SceneObjectBase { private getVizPanel() { this.lineFilterScene = new LineFilterScene(); - // If this is called before the query has executed, we need to return an empty data node to init the loading state, instead of no-data - const data = - sceneGraph.getData(this).state ?? - new SceneDataNode({ - data: { - state: LoadingState.Loading, - series: [], - timeRange: sceneGraph.getTimeRange(this).state.value, - }, - }).state; - return new SceneFlexLayout({ direction: 'column', children: @@ -188,10 +174,7 @@ export class LogsListScene extends SceneObjectBase { }), new SceneFlexItem({ height: 'calc(100vh - 220px)', - body: new LogsPanelScene({ - // this data node is just for the initial loading state before we get our first response back - data, - }), + body: new LogsPanelScene({}), }), ] : [ @@ -201,9 +184,7 @@ export class LogsListScene extends SceneObjectBase { }), new SceneFlexItem({ height: 'calc(100vh - 220px)', - body: new LogsTableScene({ - data, - }), + body: new LogsTableScene({}), }), ], }); diff --git a/src/Components/ServiceScene/LogsPanelScene.tsx b/src/Components/ServiceScene/LogsPanelScene.tsx index f9ad1e5a7..9ad4bc473 100644 --- a/src/Components/ServiceScene/LogsPanelScene.tsx +++ b/src/Components/ServiceScene/LogsPanelScene.tsx @@ -2,15 +2,12 @@ import { AdHocFiltersVariable, PanelBuilders, SceneComponentProps, - SceneDataNode, - SceneDataState, sceneGraph, SceneObjectBase, SceneObjectState, VizPanel, } from '@grafana/scenes'; -import { DataFrame, LoadingState } from '@grafana/data'; -import { getLogsPanelFrame } from './ServiceScene'; +import { DataFrame } from '@grafana/data'; import { getLogOption } from '../../services/store'; import { LogsPanelHeaderActions } from '../Table/LogsHeaderActions'; import React from 'react'; @@ -20,15 +17,13 @@ import { addToFilters, FilterType } from './Breakdowns/AddToFiltersButton'; import { getLabelTypeFromFrame, LabelType } from '../../services/fields'; import { getAdHocFiltersVariable, VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from '../../services/variables'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '../../services/analytics'; -import { areArraysEqual } from '../../services/comparison'; interface LogsPanelSceneState extends SceneObjectState { - data: SceneDataState; body?: VizPanel; } export class LogsPanelScene extends SceneObjectBase { - constructor(state: Partial & { data: SceneDataState }) { + constructor(state: Partial) { super({ ...state, }); @@ -42,52 +37,15 @@ export class LogsPanelScene extends SceneObjectBase { body: this.getLogsPanel(), }); } - - this._subs.add( - sceneGraph.getData(this).subscribeToState((newState, prevState) => { - if (!areArraysEqual(newState.data?.series, prevState.data?.series)) { - const dataFrame = getLogsPanelFrame(newState.data); - - // If we have a response, set it - if (dataFrame && newState.data) { - this.setState({ - data: newState, - }); - - // And update data node loading state - this.state.body?.state.$data?.setState({ - data: { - ...newState.data, - state: LoadingState.Done, - }, - }); - } else if (this.state.body?.state.$data && this.state.body?.state.$data.state.data) { - // otherwise set a loading state in the viz - this.state.body.state.$data.setState({ - ...this.state.body.state.$data.state, - data: { - ...this.state.body.state.$data.state.data, - state: LoadingState.Loading, - }, - }); - } - } - }) - ); } private getLogsPanel() { const parentModel = sceneGraph.getAncestor(this, LogsListScene); const visualizationType = parentModel.state.visualizationType; - const dataNode = new SceneDataNode({ - data: this.state.data?.data, - }); - return ( PanelBuilders.logs() .setTitle('Logs') - .setData(dataNode) .setOption('showTime', true) // @ts-expect-error Requires unreleased @grafana/data. Type error, doesn't cause other errors. .setOption('onClickFilterLabel', this.handleLabelFilterClick) diff --git a/src/Components/ServiceScene/LogsTableScene.tsx b/src/Components/ServiceScene/LogsTableScene.tsx index 869e418ba..1fe3a1e08 100644 --- a/src/Components/ServiceScene/LogsTableScene.tsx +++ b/src/Components/ServiceScene/LogsTableScene.tsx @@ -1,4 +1,4 @@ -import { SceneComponentProps, SceneDataState, sceneGraph, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; +import { SceneComponentProps, sceneGraph, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; import { LogsListScene } from './LogsListScene'; import { AdHocVariableFilter, LoadingState } from '@grafana/data'; import { TableProvider } from '../Table/TableProvider'; @@ -11,12 +11,11 @@ import { areArraysEqual } from '../../services/comparison'; import { getLogsPanelFrame } from './ServiceScene'; interface LogsTableSceneState extends SceneObjectState { - data: SceneDataState; loading?: LoadingState; } export class LogsTableScene extends SceneObjectBase { - constructor(state: Partial & { data: SceneDataState }) { + constructor(state: Partial) { super(state); this.addActivationHandler(this.onActivate.bind(this)); @@ -26,34 +25,16 @@ export class LogsTableScene extends SceneObjectBase { * We can't subscribe to the state of the data provider anymore, because there are multiple queries running in each data provider * So we need to manually update the data state to prevent unnecessary re-renders that cause flickering and break loading states */ - public onActivate() { - this._subs.add( - sceneGraph.getData(this).subscribeToState((newState, prevState) => { - const dataFrame = getLogsPanelFrame(newState.data); - - // Just this query is done - if (dataFrame) { - this.setState({ - data: newState, - loading: newState.data?.state, - }); - } else { - // Query is loading - this.setState({ - loading: newState.data?.state, - }); - } - }) - ); - } + public onActivate() {} public static Component = ({ model }: SceneComponentProps) => { const styles = getStyles(); // Get state from parent model const parentModel = sceneGraph.getAncestor(model, LogsListScene); + const { data } = sceneGraph.getData(model).useState(); const { selectedLine, urlColumns, visualizationType } = parentModel.useState(); // Get data state - const { data, loading } = model.useState(); + const { loading } = model.useState(); // Get time range const timeRange = sceneGraph.getTimeRange(model); @@ -80,7 +61,7 @@ export class LogsTableScene extends SceneObjectBase { } }; - const dataFrame = getLogsPanelFrame(data?.data); + const dataFrame = getLogsPanelFrame(data); return (
From 7bcf816bd4dac649fee3f9a09cc933eef9b96964 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 14:52:29 -0500 Subject: [PATCH 14/37] chore: clean up --- src/Components/ServiceScene/BreakdownViews.ts | 71 ++++++++++++++++--- .../Breakdowns/FieldsBreakdownScene.tsx | 20 ------ .../Breakdowns/LabelBreakdownScene.tsx | 10 --- .../Patterns/PatternsBreakdownScene.tsx | 14 +--- src/Components/ServiceScene/LogsListScene.tsx | 18 ----- 5 files changed, 65 insertions(+), 68 deletions(-) diff --git a/src/Components/ServiceScene/BreakdownViews.ts b/src/Components/ServiceScene/BreakdownViews.ts index e8099fb35..c9f4d1808 100644 --- a/src/Components/ServiceScene/BreakdownViews.ts +++ b/src/Components/ServiceScene/BreakdownViews.ts @@ -1,13 +1,11 @@ import { PageSlugs, ValueSlugs } from '../../services/routing'; -import { buildLogsListScene } from './LogsListScene'; +import { LogsListScene } from './LogsListScene'; import { testIds } from '../../services/testIds'; -import { buildLabelBreakdownActionScene, buildLabelValuesBreakdownActionScene } from './Breakdowns/LabelBreakdownScene'; -import { - buildFieldsBreakdownActionScene, - buildFieldValuesBreakdownActionScene, -} from './Breakdowns/FieldsBreakdownScene'; -import { buildPatternsScene } from './Breakdowns/Patterns/PatternsBreakdownScene'; -import { SceneObject } from '@grafana/scenes'; +import { buildLabelValuesBreakdownActionScene, LabelBreakdownScene } from './Breakdowns/LabelBreakdownScene'; +import { FieldsBreakdownScene } from './Breakdowns/FieldsBreakdownScene'; +import { PatternsBreakdownScene } from './Breakdowns/Patterns/PatternsBreakdownScene'; +import { SceneFlexItem, SceneFlexLayout, SceneObject } from '@grafana/scenes'; +import { LogsVolumePanel } from './LogsVolumePanel'; interface ValueBreakdownViewDefinition { displayName: string; @@ -63,3 +61,60 @@ export const valueBreakdownViews: ValueBreakdownViewDefinition[] = [ testId: testIds.exploreServiceDetails.tabFields, }, ]; + +function buildPatternsScene() { + return new SceneFlexLayout({ + children: [ + new SceneFlexItem({ + body: new PatternsBreakdownScene({}), + }), + ], + }); +} + +function buildFieldsBreakdownActionScene(changeFieldNumber: (n: string[]) => void) { + return new SceneFlexLayout({ + children: [ + new SceneFlexItem({ + body: new FieldsBreakdownScene({ changeFields: changeFieldNumber }), + }), + ], + }); +} + +function buildFieldValuesBreakdownActionScene(value: string) { + return new SceneFlexLayout({ + children: [ + new SceneFlexItem({ + body: new FieldsBreakdownScene({ value }), + }), + ], + }); +} + +function buildLogsListScene() { + return new SceneFlexLayout({ + direction: 'column', + children: [ + new SceneFlexItem({ + minHeight: 200, + body: new LogsVolumePanel({}), + }), + new SceneFlexItem({ + minHeight: '470px', + height: 'calc(100vh - 500px)', + body: new LogsListScene({}), + }), + ], + }); +} + +function buildLabelBreakdownActionScene() { + return new SceneFlexLayout({ + children: [ + new SceneFlexItem({ + body: new LabelBreakdownScene({}), + }), + ], + }); +} diff --git a/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx index 342638385..67843c155 100644 --- a/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx @@ -522,26 +522,6 @@ function getExpr(field: string) { const GRID_TEMPLATE_COLUMNS = 'repeat(auto-fit, minmax(400px, 1fr))'; -export function buildFieldsBreakdownActionScene(changeFieldNumber: (n: string[]) => void) { - return new SceneFlexLayout({ - children: [ - new SceneFlexItem({ - body: new FieldsBreakdownScene({ changeFields: changeFieldNumber }), - }), - ], - }); -} - -export function buildFieldValuesBreakdownActionScene(value: string) { - return new SceneFlexLayout({ - children: [ - new SceneFlexItem({ - body: new FieldsBreakdownScene({ value }), - }), - ], - }); -} - interface SelectLabelActionState extends SceneObjectState { labelName: string; } diff --git a/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx index 697d95bde..26dc3393e 100644 --- a/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx @@ -424,16 +424,6 @@ function getStyles(theme: GrafanaTheme2) { const GRID_TEMPLATE_COLUMNS = 'repeat(auto-fit, minmax(400px, 1fr))'; -export function buildLabelBreakdownActionScene() { - return new SceneFlexLayout({ - children: [ - new SceneFlexItem({ - body: new LabelBreakdownScene({}), - }), - ], - }); -} - export function buildLabelValuesBreakdownActionScene(value: string) { return new SceneFlexLayout({ children: [ diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index d5b199a44..d9818ceb6 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -111,10 +111,10 @@ export class PatternsBreakdownScene extends SceneObjectBase { + private onDataChange = (newState: SceneDataState, prevState: SceneDataState) => { const newFrame = getPatternsFrames(newState.data); const prevFrame = getPatternsFrames(prevState.data); if (!areArraysEqual(newFrame, prevFrame) || this.state.loading) { @@ -209,13 +209,3 @@ function getStyles(theme: GrafanaTheme2) { }), }; } - -export function buildPatternsScene() { - return new SceneFlexLayout({ - children: [ - new SceneFlexItem({ - body: new PatternsBreakdownScene({}), - }), - ], - }); -} diff --git a/src/Components/ServiceScene/LogsListScene.tsx b/src/Components/ServiceScene/LogsListScene.tsx index b1a6f312c..be17e66b1 100644 --- a/src/Components/ServiceScene/LogsListScene.tsx +++ b/src/Components/ServiceScene/LogsListScene.tsx @@ -13,7 +13,6 @@ import { import { LineFilterScene } from './LineFilterScene'; import { SelectedTableRow } from '../Table/LogLineCellComponent'; import { LogsTableScene } from './LogsTableScene'; -import { LogsVolumePanel } from './LogsVolumePanel'; import { css } from '@emotion/css'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '../../services/analytics'; import { locationService } from '@grafana/runtime'; @@ -205,23 +204,6 @@ export class LogsListScene extends SceneObjectBase { }; } -export function buildLogsListScene() { - return new SceneFlexLayout({ - direction: 'column', - children: [ - new SceneFlexItem({ - minHeight: 200, - body: new LogsVolumePanel({}), - }), - new SceneFlexItem({ - minHeight: '470px', - height: 'calc(100vh - 500px)', - body: new LogsListScene({}), - }), - ], - }); -} - const styles = { panelWrapper: css({ // If you use hover-header without any header options we must manually hide the remnants, or it shows up as a 1px line in the top-right corner of the viz From 1f4d517fc6d9b7bf33776e54174945fc5ce1f4ca Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 14:56:39 -0500 Subject: [PATCH 15/37] fix: fix table loading state --- src/Components/ServiceScene/LogsTableScene.tsx | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Components/ServiceScene/LogsTableScene.tsx b/src/Components/ServiceScene/LogsTableScene.tsx index 1fe3a1e08..485073f4d 100644 --- a/src/Components/ServiceScene/LogsTableScene.tsx +++ b/src/Components/ServiceScene/LogsTableScene.tsx @@ -1,6 +1,6 @@ import { SceneComponentProps, sceneGraph, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; import { LogsListScene } from './LogsListScene'; -import { AdHocVariableFilter, LoadingState } from '@grafana/data'; +import { AdHocVariableFilter } from '@grafana/data'; import { TableProvider } from '../Table/TableProvider'; import React, { useRef } from 'react'; import { PanelChrome } from '@grafana/ui'; @@ -10,9 +10,7 @@ import { addAdHocFilter } from './Breakdowns/AddToFiltersButton'; import { areArraysEqual } from '../../services/comparison'; import { getLogsPanelFrame } from './ServiceScene'; -interface LogsTableSceneState extends SceneObjectState { - loading?: LoadingState; -} +interface LogsTableSceneState extends SceneObjectState {} export class LogsTableScene extends SceneObjectBase { constructor(state: Partial) { @@ -33,9 +31,6 @@ export class LogsTableScene extends SceneObjectBase { const { data } = sceneGraph.getData(model).useState(); const { selectedLine, urlColumns, visualizationType } = parentModel.useState(); - // Get data state - const { loading } = model.useState(); - // Get time range const timeRange = sceneGraph.getTimeRange(model); const { value: timeRangeValue } = timeRange.useState(); @@ -66,7 +61,7 @@ export class LogsTableScene extends SceneObjectBase { return (
} > From bf8c10ac1acb77027869320dfb1533503f4a2524 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 14:58:48 -0500 Subject: [PATCH 16/37] chore: clean up --- src/Components/ServiceScene/LogsTableScene.tsx | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/Components/ServiceScene/LogsTableScene.tsx b/src/Components/ServiceScene/LogsTableScene.tsx index 485073f4d..c14e67aa0 100644 --- a/src/Components/ServiceScene/LogsTableScene.tsx +++ b/src/Components/ServiceScene/LogsTableScene.tsx @@ -1,4 +1,4 @@ -import { SceneComponentProps, sceneGraph, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; +import { SceneComponentProps, sceneGraph, SceneObjectBase } from '@grafana/scenes'; import { LogsListScene } from './LogsListScene'; import { AdHocVariableFilter } from '@grafana/data'; import { TableProvider } from '../Table/TableProvider'; @@ -10,20 +10,7 @@ import { addAdHocFilter } from './Breakdowns/AddToFiltersButton'; import { areArraysEqual } from '../../services/comparison'; import { getLogsPanelFrame } from './ServiceScene'; -interface LogsTableSceneState extends SceneObjectState {} - -export class LogsTableScene extends SceneObjectBase { - constructor(state: Partial) { - super(state); - - this.addActivationHandler(this.onActivate.bind(this)); - } - - /** - * We can't subscribe to the state of the data provider anymore, because there are multiple queries running in each data provider - * So we need to manually update the data state to prevent unnecessary re-renders that cause flickering and break loading states - */ - public onActivate() {} +export class LogsTableScene extends SceneObjectBase { public static Component = ({ model }: SceneComponentProps) => { const styles = getStyles(); // Get state from parent model From 9a9a3e032dd97cd6a761e6d5416d6d34761b5beb Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 15:00:42 -0500 Subject: [PATCH 17/37] chore: clean up --- src/Components/ServiceScene/ServiceScene.tsx | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index cd08b851e..184fe8063 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -363,23 +363,10 @@ function buildGraphScene() { }); } -// @todo move to datasource or queries? -export function getPatternsQueryRunner() { +function getPatternsQueryRunner() { return getResourceQueryRunner([buildResourceQuery(VAR_LABELS_EXPR, 'patterns', { refId: PATTERNS_QUERY_REFID })]); } function getServiceSceneQueryRunner() { return getQueryRunner([buildDataQuery(LOG_STREAM_SELECTOR_EXPR, { refId: LOGS_PANEL_QUERY_REFID })]); } -// -// function getQueryRunnerFromProvider(queryRunner: SceneDataProvider): SceneQueryRunner { -// if (queryRunner instanceof SceneQueryRunner) { -// return queryRunner; -// } -// -// if (queryRunner.state.$data instanceof SceneQueryRunner) { -// return queryRunner.state.$data; -// } -// -// throw new Error('Cannot find query runner'); -// } From 6a8241a75feda4ff5d110dd78ed0770501904137 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 15:01:29 -0500 Subject: [PATCH 18/37] chore: clean up def --- src/Components/Table/TableProvider.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Components/Table/TableProvider.tsx b/src/Components/Table/TableProvider.tsx index da12ff331..7066ba967 100644 --- a/src/Components/Table/TableProvider.tsx +++ b/src/Components/Table/TableProvider.tsx @@ -7,7 +7,7 @@ import { parseLogsFrame } from '../../services/logsFrame'; import { SelectedTableRow } from './LogLineCellComponent'; interface TableProviderProps { - dataFrame?: DataFrame; + dataFrame: DataFrame; setUrlColumns: (columns: string[]) => void; urlColumns: string[]; addFilter: (filter: AdHocVariableFilter) => void; From e8bbf8c30786aeb15e8e95a4167d6e93c999de15 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 15:05:16 -0500 Subject: [PATCH 19/37] chore: clean up datasource --- src/services/datasource.ts | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 3a0d43e1f..0bcf95b14 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -9,7 +9,7 @@ import { import { DataSourceWithBackend, getDataSourceSrv } from '@grafana/runtime'; import { RuntimeDataSource, SceneObject, sceneUtils } from '@grafana/scenes'; import { DataQuery } from '@grafana/schema'; -import { Observable, Subscriber, tap } from 'rxjs'; +import { Observable, Subscriber } from 'rxjs'; import { getDataSource } from './scenes'; import { LokiQuery } from './query'; import { PLUGIN_ID } from './routing'; @@ -58,10 +58,7 @@ class WrappedLokiDatasource extends RuntimeDataSource { } query(request: SceneDataQueryRequest): Promise | Observable { - const numberOfQueries = request.targets.length; - let numberOfQueriesThatResolved = 0; - - const observable = new Observable((subscriber) => { + return new Observable((subscriber) => { if (!request.scopedVars?.__sceneObject) { throw new Error('Scene object not found in request'); } @@ -100,19 +97,6 @@ class WrappedLokiDatasource extends RuntimeDataSource { }); }); }); - - return observable.pipe( - tap((response) => { - if (response.state === LoadingState.Done || response.state === LoadingState.Error) { - numberOfQueriesThatResolved++; - if (numberOfQueriesThatResolved < numberOfQueries) { - response.state = LoadingState.Loading; - } - } - - return response; - }) - ); } private getData( @@ -126,6 +110,10 @@ class WrappedLokiDatasource extends RuntimeDataSource { return !target.resource; }); + if (!dataQueryRequest.targets.length) { + throw new Error('No valid queries!'); + } + // query the datasource and return either observable or promise const dsResponse = ds.query(dataQueryRequest); From d2907f0d6f7f1a0d3827da27989c39f6db460831 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 15:07:32 -0500 Subject: [PATCH 20/37] chore: clean up --- src/services/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 0bcf95b14..08fe0afef 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -255,7 +255,7 @@ class WrappedLokiDatasource extends RuntimeDataSource { { name: 'volume', values: response?.data.result.map((r) => Number(r.value[1])) }, ], }); - subscriber.next({ data: [df], state: LoadingState.Done }); + subscriber.next({ data: [df] }); subscriber.complete(); }); From 8e5314cde3ec4fd6cdcc6c0ae19856e773381105 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 15:11:08 -0500 Subject: [PATCH 21/37] chore: add loading state --- src/Components/ServiceScene/ServiceScene.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index 184fe8063..98ff340be 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -253,7 +253,7 @@ export class ServiceScene extends SceneObjectBase { ]; const newState = sceneGraph.getData(this).state; const frame = getLogsPanelFrame(newState.data); - if (frame) { + if (frame && newState.data?.state === LoadingState.Done) { const res = updateParserFromDataFrame(frame, this); const fields = res.fields.filter((f) => !disabledFields.includes(f)).sort((a, b) => a.localeCompare(b)); if (!areArraysEqual(fields, this.state.fields)) { From a85142521f7b38be64f21cd083e28762e7496b59 Mon Sep 17 00:00:00 2001 From: Galen Date: Fri, 9 Aug 2024 15:16:34 -0500 Subject: [PATCH 22/37] chore: add the dang video --- playwright.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playwright.config.ts b/playwright.config.ts index e21e5ef0d..cecde2db2 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -35,7 +35,7 @@ export default defineConfig({ // Turn on when debugging local tests video: { - mode: 'on-first-retry', + mode: 'on', } }, expect: { timeout: 15000 }, From 41f90e897bbffaa7c0ff2dffde922bbf131fb14b Mon Sep 17 00:00:00 2001 From: Galen Date: Mon, 12 Aug 2024 07:46:13 -0500 Subject: [PATCH 23/37] chore: remove e2e video in ci --- playwright.config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index cecde2db2..c7dd8f133 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -34,9 +34,9 @@ export default defineConfig({ trace: 'on-first-retry', // Turn on when debugging local tests - video: { - mode: 'on', - } + // video: { + // mode: 'on', + // } }, expect: { timeout: 15000 }, From c4d63aa163741f21f110dbac70b8b080eedd65cf Mon Sep 17 00:00:00 2001 From: Galen Date: Mon, 12 Aug 2024 15:17:37 -0500 Subject: [PATCH 24/37] fix: dont run when query is already running, or we have cached state --- src/Components/ServiceScene/ServiceScene.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index 98ff340be..f1acd5773 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -142,7 +142,7 @@ export class ServiceScene extends SceneObjectBase { const slug = getDrilldownSlug(); // If we don't have a patterns count in the tabs, or we are activating the patterns scene, run the pattern query - if (this.state.patternsCount === undefined || slug === 'patterns') { + if ((this.state.patternsCount === undefined || slug === 'patterns') && !this.state.$patternsData.state.data) { this.state.$patternsData.runQueries(); } From 9733c2886585a1c3ee5b68c1d11ab4b3491eaf43 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 08:50:29 -0500 Subject: [PATCH 25/37] chore: clean up extra rendering --- .../Breakdowns/Patterns/PatternsBreakdownScene.tsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index d9818ceb6..444ca1ddd 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -64,18 +64,15 @@ export class PatternsBreakdownScene extends SceneObjectBase) => { - const { body, loading, blockingMessage } = model.useState(); + const { body, loading, blockingMessage, patternFrames } = model.useState(); const { value: timeRange } = sceneGraph.getTimeRange(model).useState(); - const logsByServiceScene = sceneGraph.getAncestor(model, ServiceScene); - const { $patternsData } = logsByServiceScene.useState(); - const patterns = getPatternsFrames($patternsData?.state.data); const styles = useStyles2(getStyles); const timeRangeTooOld = dateTime().diff(timeRange.to, 'hours') >= PATTERNS_MAX_AGE_HOURS; return (
- {!loading && !patterns && ( + {!loading && !patternFrames && (

There are no pattern matches.

@@ -89,9 +86,9 @@ export class PatternsBreakdownScene extends SceneObjectBase )} - {!loading && patterns?.length === 0 && timeRangeTooOld && } - {!loading && patterns?.length === 0 && !timeRangeTooOld && } - {!loading && patterns && patterns.length > 0 && ( + {!loading && patternFrames?.length === 0 && timeRangeTooOld && } + {!loading && patternFrames?.length === 0 && !timeRangeTooOld && } + {!loading && patternFrames && patternFrames.length > 0 && (
{body && }
)} From 9a5f2ca02fe8a40a767df7926f11fc3cd79bc29f Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 08:52:50 -0500 Subject: [PATCH 26/37] chore: remove console.warn --- .../ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index 444ca1ddd..6b8353812 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -138,7 +138,6 @@ export class PatternsBreakdownScene extends SceneObjectBase Date: Tue, 13 Aug 2024 09:01:54 -0500 Subject: [PATCH 27/37] chore: refactor, clean up, remove getPatternsFrames --- .../Patterns/PatternsBreakdownScene.tsx | 26 +++++++++---------- .../Patterns/PatternsFrameScene.tsx | 10 +++---- src/Components/ServiceScene/ServiceScene.tsx | 6 +---- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index 6b8353812..835398d34 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -16,7 +16,7 @@ import { import { Text, useStyles2 } from '@grafana/ui'; import { StatusWrapper } from 'Components/ServiceScene/Breakdowns/StatusWrapper'; import { VAR_LABEL_GROUP_BY } from 'services/variables'; -import { getPatternsFrames, ServiceScene } from '../../ServiceScene'; +import { ServiceScene } from '../../ServiceScene'; import { IndexScene } from '../../../IndexScene/IndexScene'; import { PatternsFrameScene } from './PatternsFrameScene'; import { PatternsViewTextSearch } from './PatternsViewTextSearch'; @@ -100,11 +100,11 @@ export class PatternsBreakdownScene extends SceneObjectBase { - const newFrame = getPatternsFrames(newState.data); - const prevFrame = getPatternsFrames(prevState.data); - if (!areArraysEqual(newFrame, prevFrame) || this.state.loading) { - this.updatePatternFrames(newFrame); + const newFrames = newState.data?.series; + const prevFrames = prevState.data?.series; + if (!areArraysEqual(newFrames, prevFrames) || this.state.loading) { + this.updatePatternFrames(newFrames); } }; @@ -136,12 +136,12 @@ export class PatternsBreakdownScene extends SceneObjectBase { + return dataFrame.map((dataFrame) => { const existingPattern = appliedPatterns?.find((appliedPattern) => appliedPattern.pattern === dataFrame.name); const sum: number = dataFrame.meta?.custom?.sum; diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx index cb97bbb64..174977f46 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx @@ -12,7 +12,7 @@ import { VizPanel, } from '@grafana/scenes'; import { LegendDisplayMode, PanelContext, SeriesVisibilityChangeMode } from '@grafana/ui'; -import { getPatternsFrames, ServiceScene } from '../../ServiceScene'; +import { ServiceScene } from '../../ServiceScene'; import { onPatternClick } from './FilterByPatternsButton'; import { IndexScene } from '../../../IndexScene/IndexScene'; import { PatternsViewTableScene } from './PatternsViewTableScene'; @@ -45,7 +45,7 @@ export class PatternsFrameScene extends SceneObjectBase const { body, loading } = model.useState(); const logsByServiceScene = sceneGraph.getAncestor(model, ServiceScene); const { $patternsData } = logsByServiceScene.useState(); - const patterns = getPatternsFrames($patternsData?.state.data); + const patterns = $patternsData?.state.data?.series; return (
@@ -60,8 +60,8 @@ export class PatternsFrameScene extends SceneObjectBase // If the patterns have changed, recalculate the dataframes this._subs.add( sceneGraph.getAncestor(this, ServiceScene).subscribeToState((newState, prevState) => { - const newFrame = getPatternsFrames(newState?.$patternsData?.state?.data); - const prevFrame = getPatternsFrames(prevState?.$patternsData?.state?.data); + const newFrame = newState?.$patternsData?.state?.data?.series; + const prevFrame = prevState?.$patternsData?.state?.data?.series; if (!areArraysEqual(newFrame, prevFrame)) { const patternsBreakdownScene = sceneGraph.getAncestor(this, PatternsBreakdownScene); @@ -113,7 +113,7 @@ export class PatternsFrameScene extends SceneObjectBase const serviceScene = sceneGraph.getAncestor(this, ServiceScene); - const lokiPatterns = getPatternsFrames(serviceScene.state.$patternsData?.state.data); + const lokiPatterns = serviceScene.state.$patternsData?.state.data?.series; if (!lokiPatterns || !patternFrames) { console.warn('Failed to update PatternsFrameScene body'); return; diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index f1acd5773..2bfd23711 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -62,10 +62,6 @@ export function getLogsPanelFrame(data: PanelData | undefined) { return data?.series.find((series) => series.refId === LOGS_PANEL_QUERY_REFID); } -export function getPatternsFrames(data: PanelData | undefined) { - return data?.series.filter((series) => series.refId === PATTERNS_QUERY_REFID); -} - export class ServiceScene extends SceneObjectBase { protected _variableDependency = new VariableDependencyConfig(this, { variableNames: [VAR_DATASOURCE, VAR_LABELS, VAR_FIELDS, VAR_PATTERNS, VAR_LEVELS], @@ -160,7 +156,7 @@ export class ServiceScene extends SceneObjectBase { this._subs.add( this.state.$patternsData.subscribeToState((newState) => { if (newState.data?.state === LoadingState.Done) { - const patternsResponse = getPatternsFrames(newState.data); + const patternsResponse = newState.data.series; if (patternsResponse?.length !== undefined) { // Save the count of patterns to state this.setState({ From 3d56793911d193ac2ae17e0b5b8da892b062b60e Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:05:13 -0500 Subject: [PATCH 28/37] chore: rename LogsActionBarScene to ActionBarScene --- .../{LogsActionBarScene.tsx => ActionBarScene.tsx} | 6 +++--- src/Components/ServiceScene/ServiceScene.tsx | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) rename src/Components/ServiceScene/{LogsActionBarScene.tsx => ActionBarScene.tsx} (94%) diff --git a/src/Components/ServiceScene/LogsActionBarScene.tsx b/src/Components/ServiceScene/ActionBarScene.tsx similarity index 94% rename from src/Components/ServiceScene/LogsActionBarScene.tsx rename to src/Components/ServiceScene/ActionBarScene.tsx index 857fabcde..757d6dd8e 100644 --- a/src/Components/ServiceScene/LogsActionBarScene.tsx +++ b/src/Components/ServiceScene/ActionBarScene.tsx @@ -13,10 +13,10 @@ import { GrafanaTheme2 } from '@grafana/data'; import { css } from '@emotion/css'; import { BreakdownViewDefinition, breakdownViewsDefinitions } from './BreakdownViews'; -export interface LogsActionBarSceneState extends SceneObjectState {} +export interface ActionBarSceneState extends SceneObjectState {} -export class LogsActionBarScene extends SceneObjectBase { - public static Component = ({ model }: SceneComponentProps) => { +export class ActionBarScene extends SceneObjectBase { + public static Component = ({ model }: SceneComponentProps) => { const styles = useStyles2(getStyles); const exploration = getExplorationFor(model); let currentBreakdownViewSlug = getDrilldownSlug(); diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index 2bfd23711..e8e904447 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -35,7 +35,7 @@ import { SERVICE_NAME } from 'Components/ServiceSelectionScene/ServiceSelectionS import { getMetadataService } from '../../services/metadata'; import { navigateToDrilldownPage, navigateToIndex } from '../../services/navigate'; import { areArraysEqual } from '../../services/comparison'; -import { LogsActionBarScene } from './LogsActionBarScene'; +import { ActionBarScene } from './ActionBarScene'; import { breakdownViewsDefinitions, valueBreakdownViews } from './BreakdownViews'; const LOGS_PANEL_QUERY_REFID = 'logsPanelQuery'; @@ -353,7 +353,7 @@ function buildGraphScene() { children: [ new SceneFlexItem({ ySizing: 'content', - body: new LogsActionBarScene({}), + body: new ActionBarScene({}), }), ], }); From b6cfe4d463117fa0df2a8094f86b287373d8c252 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:06:47 -0500 Subject: [PATCH 29/37] chore: refactor counter ternary --- src/Components/ServiceScene/ActionBarScene.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Components/ServiceScene/ActionBarScene.tsx b/src/Components/ServiceScene/ActionBarScene.tsx index 757d6dd8e..18c4f1098 100644 --- a/src/Components/ServiceScene/ActionBarScene.tsx +++ b/src/Components/ServiceScene/ActionBarScene.tsx @@ -52,7 +52,7 @@ export class ActionBarScene extends SceneObjectBase { key={index} label={tab.displayName} active={currentBreakdownViewSlug === tab.value} - counter={!loading ? getCounter(tab, { ...state, $data }) : undefined} + counter={loading ? undefined : getCounter(tab, { ...state, $data })} icon={loading ? 'spinner' : undefined} onChangeTab={() => { if (tab.value !== currentBreakdownViewSlug || allowNavToParent) { From e9cd5d746ad8bd97bbf5406ef483c8ab865a7982 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:07:25 -0500 Subject: [PATCH 30/37] chore: remove nested if --- .../ServiceScene/ActionBarScene.tsx | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Components/ServiceScene/ActionBarScene.tsx b/src/Components/ServiceScene/ActionBarScene.tsx index 18c4f1098..bdf1a2db1 100644 --- a/src/Components/ServiceScene/ActionBarScene.tsx +++ b/src/Components/ServiceScene/ActionBarScene.tsx @@ -55,7 +55,7 @@ export class ActionBarScene extends SceneObjectBase { counter={loading ? undefined : getCounter(tab, { ...state, $data })} icon={loading ? 'spinner' : undefined} onChangeTab={() => { - if (tab.value !== currentBreakdownViewSlug || allowNavToParent) { + if ((tab.value && tab.value !== currentBreakdownViewSlug) || allowNavToParent) { reportAppInteraction( USER_EVENTS_PAGES.service_details, USER_EVENTS_ACTIONS.service_details.action_view_changed, @@ -64,16 +64,15 @@ export class ActionBarScene extends SceneObjectBase { previousActionView: currentBreakdownViewSlug, } ); - if (tab.value) { - const serviceScene = sceneGraph.getAncestor(model, ServiceScene); - const variable = getLabelsVariable(serviceScene); - const service = variable.state.filters.find((f) => f.key === SERVICE_NAME); - if (service?.value) { - navigateToDrilldownPage(tab.value, serviceScene); - } else { - navigateToIndex(); - } + const serviceScene = sceneGraph.getAncestor(model, ServiceScene); + const variable = getLabelsVariable(serviceScene); + const service = variable.state.filters.find((f) => f.key === SERVICE_NAME); + + if (service?.value) { + navigateToDrilldownPage(tab.value, serviceScene); + } else { + navigateToIndex(); } } }} From 080bf3d904af618c2f02cee1c68b7016a321fa0a Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:12:23 -0500 Subject: [PATCH 31/37] chore: clean up --- src/services/datasource.ts | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 08fe0afef..4e0594d82 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -70,27 +70,26 @@ class WrappedLokiDatasource extends RuntimeDataSource { throw new Error('Invalid datasource!'); } - const dataQueryRequest = { ...request }; // override the target datasource to Loki - dataQueryRequest.targets = request.targets.map((target) => { + request.targets = request.targets.map((target) => { target.datasource = ds; return target; }); - dataQueryRequest.targets.forEach((target) => { + request.targets.forEach((target) => { const requestType = target?.resource; switch (requestType) { case 'volume': { - this.getVolume(dataQueryRequest, ds, subscriber); + this.getVolume(request, ds, subscriber); break; } case 'patterns': { - this.getPatterns(dataQueryRequest, ds, subscriber); + this.getPatterns(request, ds, subscriber); break; } default: { - this.getData(dataQueryRequest, ds, subscriber); + this.getData(request, ds, subscriber); break; } } @@ -104,19 +103,8 @@ class WrappedLokiDatasource extends RuntimeDataSource { ds: DataSourceWithBackend, subscriber: Subscriber ) { - const dataQueryRequest = { ...request }; - // override the target datasource to Loki - dataQueryRequest.targets = request.targets.filter((target) => { - return !target.resource; - }); - - if (!dataQueryRequest.targets.length) { - throw new Error('No valid queries!'); - } - // query the datasource and return either observable or promise - const dsResponse = ds.query(dataQueryRequest); - + const dsResponse = ds.query(request); dsResponse.subscribe(subscriber); return subscriber; From 5e5101c0f61eba72eda39584b939266fe7d178ac Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:32:47 -0500 Subject: [PATCH 32/37] chore: partition queries by resource --- src/services/datasource.ts | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 4e0594d82..82f5bf654 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -13,6 +13,7 @@ import { Observable, Subscriber } from 'rxjs'; import { getDataSource } from './scenes'; import { LokiQuery } from './query'; import { PLUGIN_ID } from './routing'; +import { partition } from 'lodash'; export const WRAPPED_LOKI_DS_UID = 'wrapped-loki-ds-uid'; @@ -76,21 +77,28 @@ class WrappedLokiDatasource extends RuntimeDataSource { return target; }); - request.targets.forEach((target) => { - const requestType = target?.resource; + const queriesPartitionedByResource = partition(request.targets, (target) => { + return target.resource; + }); - switch (requestType) { - case 'volume': { - this.getVolume(request, ds, subscriber); - break; - } - case 'patterns': { - this.getPatterns(request, ds, subscriber); - break; - } - default: { - this.getData(request, ds, subscriber); - break; + queriesPartitionedByResource.forEach((queries) => { + if (queries.length !== 0) { + // All resource strings in the partition will be the same + const requestType = queries[0]?.resource; + + switch (requestType) { + case 'volume': { + this.getVolume(request, ds, subscriber); + break; + } + case 'patterns': { + this.getPatterns(request, ds, subscriber); + break; + } + default: { + this.getData(request, ds, subscriber); + break; + } } } }); From 3101bf8fb55071c891999e7821936548c8a1f245 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:40:09 -0500 Subject: [PATCH 33/37] chore: add assertion that queries do not have multiple targets --- src/services/datasource.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 82f5bf654..449430011 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -81,6 +81,18 @@ class WrappedLokiDatasource extends RuntimeDataSource { return target.resource; }); + const resourceCount = queriesPartitionedByResource.reduce((acc, queries) => { + if (queries[0]?.resource) { + return acc++; + } + return acc; + }, 0); + + // Don't mix queries that call different endpoints! + if (resourceCount > 1) { + throw new Error('Each request can only query a single resource!'); + } + queriesPartitionedByResource.forEach((queries) => { if (queries.length !== 0) { // All resource strings in the partition will be the same From db0b7a7d122b0e53a4a6df375671d1aaf1049324 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:43:53 -0500 Subject: [PATCH 34/37] chore: refactor --- src/services/datasource.ts | 49 +++++++++++++++----------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 449430011..a46a0fea1 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -13,7 +13,6 @@ import { Observable, Subscriber } from 'rxjs'; import { getDataSource } from './scenes'; import { LokiQuery } from './query'; import { PLUGIN_ID } from './routing'; -import { partition } from 'lodash'; export const WRAPPED_LOKI_DS_UID = 'wrapped-loki-ds-uid'; @@ -77,40 +76,30 @@ class WrappedLokiDatasource extends RuntimeDataSource { return target; }); - const queriesPartitionedByResource = partition(request.targets, (target) => { - return target.resource; + const targetsSet = new Set(); + request.targets.forEach((target) => { + targetsSet.add(target.resource); }); - const resourceCount = queriesPartitionedByResource.reduce((acc, queries) => { - if (queries[0]?.resource) { - return acc++; - } - return acc; - }, 0); - - // Don't mix queries that call different endpoints! - if (resourceCount > 1) { - throw new Error('Each request can only query a single resource!'); + if (targetsSet.size !== 1) { + throw new Error('A request cannot contain queries to multiple endpoints'); } - queriesPartitionedByResource.forEach((queries) => { - if (queries.length !== 0) { - // All resource strings in the partition will be the same - const requestType = queries[0]?.resource; + request.targets.forEach((target) => { + const requestType = target?.resource; - switch (requestType) { - case 'volume': { - this.getVolume(request, ds, subscriber); - break; - } - case 'patterns': { - this.getPatterns(request, ds, subscriber); - break; - } - default: { - this.getData(request, ds, subscriber); - break; - } + switch (requestType) { + case 'volume': { + this.getVolume(request, ds, subscriber); + break; + } + case 'patterns': { + this.getPatterns(request, ds, subscriber); + break; + } + default: { + this.getData(request, ds, subscriber); + break; } } }); From 96f1daf95eecea0b685f45b08d39a523b7d2520a Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:48:01 -0500 Subject: [PATCH 35/37] chore: no need to iterate --- src/services/datasource.ts | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/services/datasource.ts b/src/services/datasource.ts index a46a0fea1..2cd661eff 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -78,31 +78,29 @@ class WrappedLokiDatasource extends RuntimeDataSource { const targetsSet = new Set(); request.targets.forEach((target) => { - targetsSet.add(target.resource); + targetsSet.add(target.resource ?? ''); }); if (targetsSet.size !== 1) { throw new Error('A request cannot contain queries to multiple endpoints'); } - request.targets.forEach((target) => { - const requestType = target?.resource; + const requestType = request.targets[0].resource; - switch (requestType) { - case 'volume': { - this.getVolume(request, ds, subscriber); - break; - } - case 'patterns': { - this.getPatterns(request, ds, subscriber); - break; - } - default: { - this.getData(request, ds, subscriber); - break; - } + switch (requestType) { + case 'volume': { + this.getVolume(request, ds, subscriber); + break; } - }); + case 'patterns': { + this.getPatterns(request, ds, subscriber); + break; + } + default: { + this.getData(request, ds, subscriber); + break; + } + } }); }); } From 24a60602998fc7078d8b01cabb99a77f660a59e1 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 09:58:26 -0500 Subject: [PATCH 36/37] chore: move patterns files --- .../Patterns/PatternsLogsSampleScene.tsx | 266 ++++++++++++++++++ .../Patterns/PatternsTableExpandedRow.tsx | 30 ++ 2 files changed, 296 insertions(+) create mode 100644 src/Components/ServiceScene/Breakdowns/Patterns/PatternsLogsSampleScene.tsx create mode 100644 src/Components/ServiceScene/Breakdowns/Patterns/PatternsTableExpandedRow.tsx diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsLogsSampleScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsLogsSampleScene.tsx new file mode 100644 index 000000000..8ef02d262 --- /dev/null +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsLogsSampleScene.tsx @@ -0,0 +1,266 @@ +import { + PanelBuilders, + SceneComponentProps, + SceneDataProviderResult, + SceneFlexItem, + SceneFlexLayout, + sceneGraph, + SceneObjectBase, + SceneObjectState, + SceneReactObject, + VizPanel, +} from '@grafana/scenes'; +import React from 'react'; + +import { LoadingState } from '@grafana/data'; +import { Alert, Button } from '@grafana/ui'; +import { + getFieldsVariable, + getLevelsVariable, + getLineFilterVariable, + LOG_STREAM_SELECTOR_EXPR, + PATTERNS_SAMPLE_SELECTOR_EXPR, + VAR_PATTERNS_EXPR, +} from '../../../../services/variables'; +import { buildDataQuery, LokiQuery, renderPatternFilters } from '../../../../services/query'; +import { getQueryRunner } from '../../../../services/panel'; +import { AppliedPattern } from '../../../IndexScene/IndexScene'; +import { PatternsViewTableScene } from './PatternsViewTableScene'; +import { emptyStateStyles } from '../FieldsBreakdownScene'; + +interface PatternsLogsSampleSceneState extends SceneObjectState { + pattern: string; + body?: SceneFlexLayout; +} +export class PatternsLogsSampleScene extends SceneObjectBase { + constructor(state: PatternsLogsSampleSceneState) { + super(state); + + this.addActivationHandler(this.onActivate.bind(this)); + } + + private onActivate() { + if (this.state.body) { + return; + } + + // We start by querying with the users current query context + const queryWithFilters = buildDataQuery(LOG_STREAM_SELECTOR_EXPR); + this.replacePatternsInQuery(queryWithFilters); + + // but if that fails to return results, we fire the query without the filters, instead of showing no-data in the viz + const queryRunnerWithFilters = getQueryRunner([queryWithFilters]); + queryRunnerWithFilters.getResultsStream().subscribe((value) => { + this.onQueryWithFiltersResult(value); + }); + + this.setState({ + body: new SceneFlexLayout({ + direction: 'column', + children: [ + new SceneFlexItem({ + body: undefined, + width: '100%', + height: 0, + }), + new SceneFlexItem({ + height: 300, + width: '100%', + body: PanelBuilders.logs() + .setHoverHeader(true) + .setOption('showLogContextToggle', true) + .setOption('showTime', true) + .setData(queryRunnerWithFilters) + .build(), + }), + ], + }), + }); + } + + private replacePatternsInQuery(queryWithFilters: LokiQuery) { + const pendingPattern: AppliedPattern = { + pattern: this.state.pattern, + type: 'include', + }; + const patternsLine = renderPatternFilters([pendingPattern]); + queryWithFilters.expr = queryWithFilters.expr.replace(VAR_PATTERNS_EXPR, patternsLine); + } + + private clearFilters = () => { + const filterVariable = getFieldsVariable(this); + const lineFilterVariable = getLineFilterVariable(this); + const levelsVariable = getLevelsVariable(this); + filterVariable.setState({ + filters: [], + }); + levelsVariable.setState({ + filters: [], + }); + if (lineFilterVariable.state.value) { + lineFilterVariable.changeValueTo(''); + + const noticeFlexItem = this.getNoticeFlexItem(); + + // The query we just fired is already correct after we clear the filters, we just need to hide the warning, and allow filtering + noticeFlexItem?.setState({ + isHidden: true, + }); + + this.removePatternFromFilterExclusion(); + } + }; + + private removePatternFromFilterExclusion() { + const patternsViewTableScene = sceneGraph.getAncestor(this, PatternsViewTableScene); + const patternsNotMatchingFilters = patternsViewTableScene.state.patternsNotMatchingFilters ?? []; + + const index = patternsNotMatchingFilters.findIndex((pattern) => pattern === this.state.pattern); + + if (index !== -1) { + patternsNotMatchingFilters.splice(index, 1); + // remove this pattern, as they can filter by this pattern again + patternsViewTableScene.setState({ + patternsNotMatchingFilters: patternsNotMatchingFilters, + }); + } + } + + /** + * If the first query with the users filters applied fails, we run another one after removing the filters + * @param value + */ + private onQueryError = (value: SceneDataProviderResult) => { + if ( + (value.data.state === LoadingState.Done && + (value.data.series.length === 0 || value.data.series.every((frame) => frame.length === 0))) || + value.data.state === LoadingState.Error + ) { + // Logging an error so loki folks can debug why some patterns returned from the API seem to fail. + console.error('Pattern sample query returns no results', { + pattern: this.state.pattern, + traceIds: value.data.traceIds, + request: value.data.request, + }); + + this.setWarningMessage( + + This pattern returns no logs. + + ); + + const panelFlexItem = this.getVizFlexItem(); + + // Run another query without the filters so we can still show log lines of what the pattern looks like. + if (panelFlexItem instanceof SceneFlexItem) { + panelFlexItem.setState({ + isHidden: true, + }); + } + } + }; + + private setWarningMessage(reactNode: React.ReactNode) { + const noticeFlexItem = this.getNoticeFlexItem(); + const vizFlexItem = this.getVizFlexItem(); + + if (noticeFlexItem instanceof SceneFlexItem) { + noticeFlexItem.setState({ + isHidden: false, + height: 'auto', + body: new SceneReactObject({ + reactNode: reactNode, + }), + }); + } + return vizFlexItem; + } + + private getNoticeFlexItem() { + const children = this.getFlexItemChildren(); + return children?.[0]; + } + private getVizFlexItem() { + const children = this.getFlexItemChildren(); + return children?.[1]; + } + + private getFlexItemChildren() { + return this.state.body?.state.children; + } + + /** + * Callback to subscription of pattern sample query with all of the current query filters applied. + * If this query fails to return data, we show a warning, and attempt the pattern sample query again without applying the existing filters. + * We also add the pattern to the state of the PatternsTableViewScene so we can hide the filter buttons for this pattern, as including it would break the query + * @param value + */ + private onQueryWithFiltersResult = (value: SceneDataProviderResult) => { + const queryWithoutFilters = buildDataQuery(PATTERNS_SAMPLE_SELECTOR_EXPR); + this.replacePatternsInQuery(queryWithoutFilters); + + const queryRunnerWithoutFilters = getQueryRunner([queryWithoutFilters]); + + // Subscribe to the secondary query, so we can log errors and update the UI + queryRunnerWithoutFilters.getResultsStream().subscribe(this.onQueryError); + + if ( + value.data.state === LoadingState.Done && + (value.data.series.length === 0 || value.data.series.every((frame) => frame.length === 0)) + ) { + const noticeFlexItem = this.getNoticeFlexItem(); + const panelFlexItem = this.getVizFlexItem(); + + // Add a warning notice that the patterns shown will not show up in their current log results due to their existing filters. + if (noticeFlexItem instanceof SceneFlexItem) { + noticeFlexItem.setState({ + isHidden: false, + height: 'auto', + body: new SceneReactObject({ + reactNode: ( + + The logs returned by this pattern do not match the current query filters. + + + ), + }), + }); + } + + // Run another query without the filters so we can still show log lines of what the pattern looks like. + if (panelFlexItem instanceof SceneFlexItem) { + const panel = panelFlexItem.state.body; + if (panel instanceof VizPanel) { + panel?.setState({ + $data: queryRunnerWithoutFilters, + }); + } + } + this.excludeThisPatternFromFiltering(); + } + + if (value.data.state === LoadingState.Error) { + this.onQueryError(value); + } + }; + + private excludeThisPatternFromFiltering() { + const patternsViewTableScene = sceneGraph.getAncestor(this, PatternsViewTableScene); + const patternsThatDontMatchCurrentFilters = patternsViewTableScene.state.patternsNotMatchingFilters ?? []; + + // Add this pattern to the array of patterns that don't match current filters + patternsViewTableScene.setState({ + patternsNotMatchingFilters: [...patternsThatDontMatchCurrentFilters, this.state.pattern], + }); + } + + public static Component({ model }: SceneComponentProps) { + const { body } = model.useState(); + if (body) { + return ; + } + return null; + } +} diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsTableExpandedRow.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsTableExpandedRow.tsx new file mode 100644 index 000000000..e0f66b489 --- /dev/null +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsTableExpandedRow.tsx @@ -0,0 +1,30 @@ +import React, { useEffect } from 'react'; +import { PatternsLogsSampleScene } from './PatternsLogsSampleScene'; +import { PatternsTableCellData, PatternsViewTableScene } from './PatternsViewTableScene'; + +interface ExpandedRowProps { + tableViz: PatternsViewTableScene; + row: PatternsTableCellData; +} + +export function PatternsTableExpandedRow({ tableViz, row }: ExpandedRowProps) { + const { expandedRows } = tableViz.useState(); + + const rowScene = expandedRows?.find((scene) => scene.state.key === row.pattern); + + useEffect(() => { + if (!rowScene) { + const newRowScene = buildExpandedRowScene(row.pattern); + tableViz.setState({ expandedRows: [...(tableViz.state.expandedRows ?? []), newRowScene] }); + } + }, [row, tableViz, rowScene]); + + return rowScene ? : null; +} + +function buildExpandedRowScene(pattern: string) { + return new PatternsLogsSampleScene({ + pattern: pattern, + key: pattern, + }); +} From 4508fc2ca891ed7e0767a5abfcbe4870832f2e38 Mon Sep 17 00:00:00 2001 From: Galen Date: Tue, 13 Aug 2024 10:30:24 -0500 Subject: [PATCH 37/37] chore: update types, prevent runtime error --- .../Patterns/PatternsBreakdownScene.tsx | 2 +- src/Components/ServiceScene/ServiceScene.tsx | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx index 835398d34..d902f56f5 100644 --- a/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx @@ -108,7 +108,7 @@ export class PatternsBreakdownScene extends SceneObjectBase { diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index e8e904447..50979cb4b 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -54,8 +54,8 @@ export interface ServiceSceneCustomState { export interface ServiceSceneState extends SceneObjectState, ServiceSceneCustomState { body: SceneFlexLayout | undefined; drillDownLabel?: string; - $data: SceneDataProvider; - $patternsData: SceneQueryRunner; + $data: SceneDataProvider | undefined; + $patternsData: SceneQueryRunner | undefined; } export function getLogsPanelFrame(data: PanelData | undefined) { @@ -138,12 +138,12 @@ export class ServiceScene extends SceneObjectBase { const slug = getDrilldownSlug(); // If we don't have a patterns count in the tabs, or we are activating the patterns scene, run the pattern query - if ((this.state.patternsCount === undefined || slug === 'patterns') && !this.state.$patternsData.state.data) { - this.state.$patternsData.runQueries(); + if ((this.state.patternsCount === undefined || slug === 'patterns') && !this.state.$patternsData?.state.data) { + this.state.$patternsData?.runQueries(); } this._subs.add( - this.state.$data.subscribeToState((newState) => { + this.state.$data?.subscribeToState((newState) => { if (newState.data?.state === LoadingState.Done) { const logsPanelResponse = getLogsPanelFrame(newState.data); if (logsPanelResponse) { @@ -154,7 +154,7 @@ export class ServiceScene extends SceneObjectBase { ); this._subs.add( - this.state.$patternsData.subscribeToState((newState) => { + this.state.$patternsData?.subscribeToState((newState) => { if (newState.data?.state === LoadingState.Done) { const patternsResponse = newState.data.series; if (patternsResponse?.length !== undefined) { @@ -174,7 +174,7 @@ export class ServiceScene extends SceneObjectBase { this._subs.add( labels.subscribeToState((newState, prevState) => { if (!areArraysEqual(newState.filters, prevState.filters)) { - this.state.$patternsData.runQueries(); + this.state.$patternsData?.runQueries(); } }) ); @@ -183,7 +183,7 @@ export class ServiceScene extends SceneObjectBase { this._subs.add( sceneGraph.getTimeRange(this).subscribeToState(() => { this.updateLabels(); - this.state.$patternsData.runQueries(); + this.state.$patternsData?.runQueries(); }) ); }