Patterns: Use runtime datasource for queries

playwright.config.ts

@@ -34,9 +34,9 @@ export default defineConfig<PluginOptions>({
     trace: 'on-first-retry',
 
     // Turn on when debugging local tests
-    // video: {
-    //   mode: 'on',
-    // }
+    video: {
+      mode: 'on-first-retry',
+    }
   },
   expect: { timeout: 15000 },
 

project-words.txt

@@ -469,4 +469,5 @@ subqueries
 dbscan
 svgs
 Dont
+REFID
 unroute

src/Components/IndexScene/IndexScene.tsx

@@ -161,7 +161,9 @@ function getContentScene(drillDownLabel?: string) {
     return new ServiceSelectionScene({});
   }
 
-  return new ServiceScene({ drillDownLabel });
+  return new ServiceScene({
+    drillDownLabel,
+  });
 }
 
 function getVariableSet(initialDatasourceUid: string, initialFilters?: AdHocVariableFilter[]) {

src/Components/ServiceScene/BreakdownViews.ts

@@ -0,0 +1,65 @@
+import { PageSlugs, ValueSlugs } from '../../services/routing';
+import { buildLogsListScene } from './LogsListScene';
+import { testIds } from '../../services/testIds';
+import { buildLabelBreakdownActionScene, buildLabelValuesBreakdownActionScene } from './Breakdowns/LabelBreakdownScene';
+import {
+  buildFieldsBreakdownActionScene,
+  buildFieldValuesBreakdownActionScene,
+} from './Breakdowns/FieldsBreakdownScene';
+import { buildPatternsScene } from './Breakdowns/Patterns/PatternsBreakdownScene';
+import { SceneObject } from '@grafana/scenes';
+
+interface ValueBreakdownViewDefinition {
+  displayName: string;
+  value: ValueSlugs;
+  testId: string;
+  getScene: (value: string) => SceneObject;
+}
+
+export interface BreakdownViewDefinition {
+  displayName: string;
+  value: PageSlugs;
+  testId: string;
+  getScene: (changeFields: (f: string[]) => void) => SceneObject;
+}
+
+export const breakdownViewsDefinitions: BreakdownViewDefinition[] = [
+  {
+    displayName: 'Logs',
+    value: PageSlugs.logs,
+    getScene: () => buildLogsListScene(),
+    testId: testIds.exploreServiceDetails.tabLogs,
+  },
+  {
+    displayName: 'Labels',
+    value: PageSlugs.labels,
+    getScene: () => buildLabelBreakdownActionScene(),
+    testId: testIds.exploreServiceDetails.tabLabels,
+  },
+  {
+    displayName: 'Fields',
+    value: PageSlugs.fields,
+    getScene: (f) => buildFieldsBreakdownActionScene(f),
+    testId: testIds.exploreServiceDetails.tabFields,
+  },
+  {
+    displayName: 'Patterns',
+    value: PageSlugs.patterns,
+    getScene: () => buildPatternsScene(),
+    testId: testIds.exploreServiceDetails.tabPatterns,
+  },
+];
+export const valueBreakdownViews: ValueBreakdownViewDefinition[] = [
+  {
+    displayName: 'Label',
+    value: ValueSlugs.label,
+    getScene: (value: string) => buildLabelValuesBreakdownActionScene(value),
+    testId: testIds.exploreServiceDetails.tabLabels,
+  },
+  {
+    displayName: 'Field',
+    value: ValueSlugs.field,
+    getScene: (value: string) => buildFieldValuesBreakdownActionScene(value),
+    testId: testIds.exploreServiceDetails.tabFields,
+  },
+];

src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx

@@ -287,7 +287,7 @@ export class FieldsBreakdownScene extends SceneObjectBase<FieldsBreakdownSceneSt
         legendFormat: `{{${optionValue}}}`,
         refId: optionValue,
       });
-      const queryRunner = getQueryRunner(query);
+      const queryRunner = getQueryRunner([query]);
       let body = PanelBuilders.timeseries().setTitle(optionValue).setData(queryRunner);
 
       if (!isAvgField(optionValue)) {
@@ -348,7 +348,7 @@ export class FieldsBreakdownScene extends SceneObjectBase<FieldsBreakdownSceneSt
     const getFilter = () => this.state.search.state.filter ?? '';
 
     return new LayoutSwitcher({
-      $data: getQueryRunner(query),
+      $data: getQueryRunner([query]),
       options: [
         { value: 'single', label: 'Single' },
         { value: 'grid', label: 'Grid' },

src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx

@@ -218,9 +218,9 @@ export class LabelBreakdownScene extends SceneObjectBase<LabelBreakdownSceneStat
           body: PanelBuilders.timeseries()
             .setTitle(optionValue)
             .setData(
-              getQueryRunner(
-                buildDataQuery(getTimeSeriesExpr(this, optionValue), { legendFormat: `{{${optionValue}}}` })
-              )
+              getQueryRunner([
+                buildDataQuery(getTimeSeriesExpr(this, optionValue), { legendFormat: `{{${optionValue}}}` }),
+              ])
             )
             .setHeaderActions(new SelectLabelAction({ labelName: optionValue }))
             .setCustomFieldConfig('stacking', { mode: StackingMode.Normal })
@@ -276,7 +276,7 @@ export class LabelBreakdownScene extends SceneObjectBase<LabelBreakdownSceneStat
     const getFilter = () => this.state.search.state.filter ?? '';
 
     return new LayoutSwitcher({
-      $data: getQueryRunner(query),
+      $data: getQueryRunner([query]),
       options: [
         { value: 'single', label: 'Single' },
         { value: 'grid', label: 'Grid' },

src/Components/ServiceScene/Breakdowns/Patterns/PatternsBreakdownScene.tsx

@@ -1,10 +1,11 @@
 import { css } from '@emotion/css';
 import React from 'react';
 
-import { DataFrame, dateTime, FieldType, GrafanaTheme2 } from '@grafana/data';
+import { DataFrame, dateTime, GrafanaTheme2 } from '@grafana/data';
 import {
   CustomVariable,
   SceneComponentProps,
+  SceneDataState,
   SceneFlexItem,
   SceneFlexLayout,
   sceneGraph,
@@ -15,7 +16,7 @@ import {
 import { Text, useStyles2 } from '@grafana/ui';
 import { StatusWrapper } from 'Components/ServiceScene/Breakdowns/StatusWrapper';
 import { VAR_LABEL_GROUP_BY } from 'services/variables';
-import { LokiPattern, ServiceScene } from '../../ServiceScene';
+import { getPatternsFrames, ServiceScene } from '../../ServiceScene';
 import { IndexScene } from '../../../IndexScene/IndexScene';
 import { PatternsFrameScene } from './PatternsFrameScene';
 import { PatternsViewTextSearch } from './PatternsViewTextSearch';
@@ -53,8 +54,8 @@ export class PatternsBreakdownScene extends SceneObjectBase<PatternsBreakdownSce
         new SceneVariableSet({
           variables: [new CustomVariable({ name: VAR_LABEL_GROUP_BY, defaultToAll: true, includeAll: true })],
         }),
-      patternFilter: '',
       loading: true,
+      patternFilter: '',
       ...state,
     });
 
@@ -66,7 +67,8 @@ export class PatternsBreakdownScene extends SceneObjectBase<PatternsBreakdownSce
     const { body, loading, blockingMessage } = model.useState();
     const { value: timeRange } = sceneGraph.getTimeRange(model).useState();
     const logsByServiceScene = sceneGraph.getAncestor(model, ServiceScene);
-    const { patterns } = logsByServiceScene.useState();
+    const { $patternsData } = logsByServiceScene.useState();
+    const patterns = getPatternsFrames($patternsData?.state.data);
     const styles = useStyles2(getStyles);
     const timeRangeTooOld = dateTime().diff(timeRange.to, 'hours') >= PATTERNS_MAX_AGE_HOURS;
 
@@ -101,21 +103,25 @@ export class PatternsBreakdownScene extends SceneObjectBase<PatternsBreakdownSce
     const serviceScene = sceneGraph.getAncestor(this, ServiceScene);
     this.setBody();
 
+    const patterns = getPatternsFrames(serviceScene.state.$patternsData?.state.data);
+
     // If the patterns exist already, update the dataframe
-    if (serviceScene.state.patterns) {
-      this.updatePatternFrames(serviceScene.state.patterns);
+    if (patterns) {
+      this.updatePatternFrames(patterns);
     }
 
     // Subscribe to changes from pattern API call
-    this._subs.add(
-      serviceScene.subscribeToState((newState, prevState) => {
-        if (!areArraysEqual(newState.patterns, prevState.patterns)) {
-          this.updatePatternFrames(newState.patterns);
-        }
-      })
-    );
+    this._subs.add(serviceScene.state.$patternsData.subscribeToState(this.onDataProviderChange));
   }
 
+  private onDataProviderChange = (newState: SceneDataState, prevState: SceneDataState) => {
+    const newFrame = getPatternsFrames(newState.data);
+    const prevFrame = getPatternsFrames(prevState.data);
+    if (!areArraysEqual(newFrame, prevFrame) || this.state.loading) {
+      this.updatePatternFrames(newFrame);
+    }
+  };
+
   private setBody() {
     this.setState({
       body: new SceneFlexLayout({
@@ -133,74 +139,37 @@ export class PatternsBreakdownScene extends SceneObjectBase<PatternsBreakdownSce
     });
   }
 
-  private updatePatternFrames(lokiPatterns?: LokiPattern[]) {
+  private updatePatternFrames(lokiPatterns?: DataFrame[]) {
     if (!lokiPatterns) {
       console.warn('failed to update pattern frames');
       return;
     }
 
     const patternFrames = this.buildPatterns(lokiPatterns);
+
     this.setState({
       patternFrames,
       loading: false,
     });
   }
 
-  private buildPatterns(patterns: LokiPattern[]): PatternFrame[] {
+  private buildPatterns(patterns: DataFrame[]): PatternFrame[] {
     const serviceScene = sceneGraph.getAncestor(this, ServiceScene);
     const appliedPatterns = sceneGraph.getAncestor(serviceScene, IndexScene).state.patterns;
 
-    let maxValue = -Infinity;
-    let minValue = 0;
-
-    return patterns
-      .map((pat) => {
-        const timeValues: number[] = [];
-        const sampleValues: number[] = [];
-        let sum = 0;
-        pat.samples.forEach(([time, value]) => {
-          timeValues.push(time * 1000);
-          const sample = parseFloat(value);
-          sampleValues.push(sample);
-          if (sample > maxValue) {
-            maxValue = sample;
-          }
-          if (sample < minValue) {
-            minValue = sample;
-          }
-          sum += sample;
-        });
-        const dataFrame: DataFrame = {
-          refId: pat.pattern,
-          fields: [
-            {
-              name: 'time',
-              type: FieldType.time,
-              values: timeValues,
-              config: {},
-            },
-            {
-              name: pat.pattern,
-              type: FieldType.number,
-              values: sampleValues,
-              config: {},
-            },
-          ],
-          length: pat.samples.length,
-          meta: {
-            preferredVisualisationType: 'graph',
-          },
-        };
-        const existingPattern = appliedPatterns?.find((appliedPattern) => appliedPattern.pattern === pat.pattern);
-
-        return {
-          dataFrame,
-          pattern: pat.pattern,
-          sum,
-          status: existingPattern?.type,
-        };
-      })
-      .sort((a, b) => b.sum - a.sum);
+    return patterns.map((dataFrame) => {
+      const existingPattern = appliedPatterns?.find((appliedPattern) => appliedPattern.pattern === dataFrame.name);
+
+      const sum: number = dataFrame.meta?.custom?.sum;
+      const patternFrame: PatternFrame = {
+        dataFrame,
+        pattern: dataFrame.name ?? '',
+        sum,
+        status: existingPattern?.type,
+      };
+
+      return patternFrame;
+    });
   }
 }
 

src/Components/ServiceScene/Breakdowns/Patterns/PatternsFrameScene.tsx

@@ -12,7 +12,7 @@ import {
   VizPanel,
 } from '@grafana/scenes';
 import { LegendDisplayMode, PanelContext, SeriesVisibilityChangeMode } from '@grafana/ui';
-import { ServiceScene } from '../../ServiceScene';
+import { getPatternsFrames, ServiceScene } from '../../ServiceScene';
 import { onPatternClick } from './FilterByPatternsButton';
 import { IndexScene } from '../../../IndexScene/IndexScene';
 import { PatternsViewTableScene } from './PatternsViewTableScene';
@@ -44,7 +44,9 @@ export class PatternsFrameScene extends SceneObjectBase<PatternsFrameSceneState>
   public static Component = ({ model }: SceneComponentProps<PatternsFrameScene>) => {
     const { body, loading } = model.useState();
     const logsByServiceScene = sceneGraph.getAncestor(model, ServiceScene);
-    const { patterns } = logsByServiceScene.useState();
+    const { $patternsData } = logsByServiceScene.useState();
+    const patterns = getPatternsFrames($patternsData?.state.data);
+
     return (
       <div className={styles.container}>
         {!loading && patterns && patterns.length > 0 && <>{body && <body.Component model={body} />}</>}
@@ -58,7 +60,10 @@ export class PatternsFrameScene extends SceneObjectBase<PatternsFrameSceneState>
     // If the patterns have changed, recalculate the dataframes
     this._subs.add(
       sceneGraph.getAncestor(this, ServiceScene).subscribeToState((newState, prevState) => {
-        if (!areArraysEqual(newState.patterns, prevState.patterns)) {
+        const newFrame = getPatternsFrames(newState?.$patternsData?.state?.data);
+        const prevFrame = getPatternsFrames(prevState?.$patternsData?.state?.data);
+
+        if (!areArraysEqual(newFrame, prevFrame)) {
           const patternsBreakdownScene = sceneGraph.getAncestor(this, PatternsBreakdownScene);
           this.updatePatterns(patternsBreakdownScene.state.patternFrames);
 
@@ -107,7 +112,8 @@ export class PatternsFrameScene extends SceneObjectBase<PatternsFrameSceneState>
     const patternFrames = patternsBreakdownScene.state.patternFrames;
 
     const serviceScene = sceneGraph.getAncestor(this, ServiceScene);
-    const lokiPatterns = serviceScene.state.patterns;
+
+    const lokiPatterns = getPatternsFrames(serviceScene.state.$patternsData?.state.data);
     if (!lokiPatterns || !patternFrames) {
       console.warn('Failed to update PatternsFrameScene body');
       return;

src/Components/ServiceScene/Breakdowns/PatternsLogsSampleScene.tsx

@@ -48,7 +48,7 @@ export class PatternsLogsSampleScene extends SceneObjectBase<PatternsLogsSampleS
     this.replacePatternsInQuery(queryWithFilters);
 
     // but if that fails to return results, we fire the query without the filters, instead of showing no-data in the viz
-    const queryRunnerWithFilters = getQueryRunner(queryWithFilters);
+    const queryRunnerWithFilters = getQueryRunner([queryWithFilters]);
     queryRunnerWithFilters.getResultsStream().subscribe((value) => {
       this.onQueryWithFiltersResult(value);
     });
@@ -198,7 +198,7 @@ export class PatternsLogsSampleScene extends SceneObjectBase<PatternsLogsSampleS
     const queryWithoutFilters = buildDataQuery(PATTERNS_SAMPLE_SELECTOR_EXPR);
     this.replacePatternsInQuery(queryWithoutFilters);
 
-    const queryRunnerWithoutFilters = getQueryRunner(queryWithoutFilters);
+    const queryRunnerWithoutFilters = getQueryRunner([queryWithoutFilters]);
 
     // Subscribe to the secondary query, so we can log errors and update the UI
     queryRunnerWithoutFilters.getResultsStream().subscribe(this.onQueryError);