Labels: Fix labels list not updating when detected_labels loads while user is viewing another tab

docker-compose.dev.yaml

@@ -3,7 +3,6 @@ version: '3.0'
 services:
   grafana:
     container_name: 'grafana-logsapp'
-    platform: 'linux/amd64'
     environment:
       - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall lokiLogsDataplane
     build:

generator/flog/log.go

@@ -22,8 +22,8 @@ const (
 	RFC5424Log = "<%d>%d %s %s %s %d ID%d %s %s"
 	// CommonLogFormat : {host} {user-identifier} {auth-user-id} [{datetime}] "{method} {request} {protocol}" {response-code} {bytes}
 	CommonLogFormat = "%s - %s [%s] \"%s %s %s\" %d %d"
-	// JSONLogFormat : {"host": "{host}", "user-identifier": "{user-identifier}", "datetime": "{datetime}", "method": "{method}", "request": "{request}", "protocol": "{protocol}", "status", {status}, "bytes": {bytes}, "referer": "{referer}"}
-	JSONLogFormat = `{"host":"%s", "user-identifier":"%s", "datetime":"%s", "method": "%s", "request": "%s", "protocol":"%s", "status":%d, "bytes":%d, "referer": "%s"}`
+	// JSONLogFormat : {"host": "{host}", "user-identifier": "{user-identifier}", "datetime": "{datetime}", "method": "{method}", "request": "{request}", "protocol": "{protocol}", "status": {status}, "bytes": {bytes}, "referer": "{referer}", "_25values": "{_25values}"
+	JSONLogFormat = `{"host":"%s", "user-identifier":"%s", "datetime":"%s", "method": "%s", "request": "%s", "protocol":"%s", "status":%d, "bytes":%d, "referer": "%s", "_25values": %d}`
 )
 
 // NewApacheCommonLog creates a log string with apache common log format
@@ -142,5 +142,6 @@ func NewJSONLogFormat(t time.Time, URI string, statusCode int) string {
 		statusCode,
 		gofakeit.Number(0, 30000),
 		gofakeit.URL(),
+		gofakeit.Number(0, 25),
 	)
 }

generator/generator.go

@@ -110,7 +110,7 @@ var generators = map[model.LabelValue]map[model.LabelValue]LogGenerator{
 				for ctx.Err() == nil {
 					level := randLevel()
 					t := time.Now()
-					if rand.Intn(10)%2 == 0 && level == ERROR {
+					if level == ERROR {
 						log := flog.NewCommonLogFormat(t, randURI(), statusFromLevel(level))
 						// Add a stacktrace to the logfmt log, and include a field that will conflict with stream selectors
 						logger.LogWithMetadata(level, t, fmt.Sprintf("%s %s", log, `method=GET namespace=whoopsie caller=flush.go:253 stacktrace="Exception in thread \"main\" java.lang.NullPointerException\n        at com.example.myproject.Book.getTitle(Book.java:16)\n        at com.example.myproject.Author.getBookTitles(Author.java:25)\n        at com.example.myproject.Bootstrap.main(Bootstrap.java:14)"`), metadata)
@@ -223,7 +223,6 @@ func startFailingMimirPod(ctx context.Context, logger Logger) {
 		"cluster":      model.LabelValue(clusters[0]),
 		"namespace":    model.LabelValue("mimir"),
 		"service_name": "mimir-ingester",
-		"pod":          "mimir-ingester" + "-" + model.LabelValue(randSeq(5)),
 	}, logger)
 
 	go func() {

src/Components/ServiceScene/Breakdowns/LabelsAggregatedBreakdownScene.tsx

@@ -9,12 +9,18 @@ import {
   SceneObjectBase,
   SceneObjectState,
   SceneQueryRunner,
+  VariableValueOption,
   VizPanel,
 } from '@grafana/scenes';
 import { LayoutSwitcher } from './LayoutSwitcher';
 import { DrawStyle, LoadingPlaceholder, StackingMode } from '@grafana/ui';
 import { getQueryRunner, setLevelColorOverrides } from '../../../services/panel';
-import { ALL_VARIABLE_VALUE, getFieldsVariable, getLabelGroupByVariable } from '../../../services/variables';
+import {
+  ALL_VARIABLE_VALUE,
+  getFieldsVariable,
+  getLabelGroupByVariable,
+  LEVEL_VARIABLE_VALUE,
+} from '../../../services/variables';
 import React from 'react';
 import { LabelBreakdownScene } from './LabelBreakdownScene';
 import { SelectLabelActionScene } from './SelectLabelActionScene';
@@ -23,6 +29,8 @@ import { limitMaxNumberOfSeriesForPanel, MAX_NUMBER_OF_TIME_SERIES } from './Tim
 import { limitFramesTransformation } from './FieldsAggregatedBreakdownScene';
 import { LokiQuery } from '../../../services/query';
 import { buildLabelsQuery, LABEL_BREAKDOWN_GRID_TEMPLATE_COLUMNS } from '../../../services/labels';
+import { ServiceScene } from '../ServiceScene';
+import { DataFrame, LoadingState } from '@grafana/data';
 
 export interface LabelsAggregatedBreakdownSceneState extends SceneObjectState {
   body?: LayoutSwitcher;
@@ -39,12 +47,30 @@ export class LabelsAggregatedBreakdownScene extends SceneObjectBase<LabelsAggreg
 
   onActivate() {
     const fields = getFieldsVariable(this);
-    this.setState({
-      body: this.build(),
-    });
+    const serviceScene = sceneGraph.getAncestor(this, ServiceScene);
+    const $detectedLabels = serviceScene.state.$detectedLabelsData;
+
+    // If the body hasn't been built yet, build it
+    if (!this.state.body) {
+      this.setState({
+        body: this.build(),
+      });
+    }
+    // Otherwise if we have the detected labels done loading, update the body
+    else if ($detectedLabels?.state.data?.state === LoadingState.Done) {
+      this.update($detectedLabels?.state.data.series[0]);
+    }
 
     this._subs.add(
-      fields.subscribeToState((newState, prevState) => {
+      $detectedLabels?.subscribeToState((newState, prevState) => {
+        if (newState.data?.state === LoadingState.Done) {
+          this.update(newState.data.series[0]);
+        }
+      })
+    );
+
+    this._subs.add(
+      fields.subscribeToState(() => {
         this.updateQueriesOnFieldsVariableChange();
       })
     );
@@ -55,10 +81,7 @@ export class LabelsAggregatedBreakdownScene extends SceneObjectBase<LabelsAggreg
       const layout = layoutObj as SceneCSSGridLayout;
       // Iterate through the existing panels
       for (let i = 0; i < layout.state.children.length; i++) {
-        const gridItem = layout.state.children[i] as SceneCSSGridItem;
-        const panel = gridItem.state.body as VizPanel;
-
-        const title = panel.state.title;
+        const { panel, title } = this.getPanelByIndex(layout, i);
         const queryRunner: SceneDataProvider | SceneQueryRunner | undefined = panel.state.$data;
         const query = buildLabelsQuery(this, title, title);
 
@@ -76,36 +99,86 @@ export class LabelsAggregatedBreakdownScene extends SceneObjectBase<LabelsAggreg
     });
   };
 
+  private getPanelByIndex(layout: SceneCSSGridLayout, i: number) {
+    const gridItem = layout.state.children[i] as SceneCSSGridItem;
+    const panel = gridItem.state.body as VizPanel;
+
+    const title = panel.state.title;
+    return { panel, title };
+  }
+
+  private update(detectedLabelsFrame: DataFrame) {
+    const variable = getLabelGroupByVariable(this);
+    const newLabels = variable.state.options.filter((opt) => opt.value !== ALL_VARIABLE_VALUE).map((opt) => opt.label);
+
+    this.state.body?.state.layouts.forEach((layoutObj) => {
+      let existingLabels = [];
+      const layout = layoutObj as SceneCSSGridLayout;
+      const newLabelsSet = new Set<string>(newLabels);
+      const updatedChildren = layout.state.children as SceneCSSGridItem[];
+
+      for (let i = 0; i < updatedChildren.length; i++) {
+        const { title } = this.getPanelByIndex(layout, i);
+
+        if (newLabelsSet.has(title)) {
+          // If the new response has this field, delete it from the set, but leave it in the layout
+          newLabelsSet.delete(title);
+        } else {
+          // Otherwise if the panel doesn't exist in the response, delete it from the layout
+          updatedChildren.splice(i, 1);
+          // And make sure to update the index, or we'll skip the next one
+          i--;
+        }
+        existingLabels.push(title);
+      }
+
+      const labelsToAdd = Array.from(newLabelsSet);
+
+      const options = labelsToAdd.map((fieldName) => {
+        return {
+          label: fieldName,
+          value: fieldName,
+        };
+      });
+
+      updatedChildren.push(...this.buildChildren(options));
+
+      const cardinalityMap = this.calculateCardinalityMap(detectedLabelsFrame);
+      updatedChildren.sort(this.sortChildren(cardinalityMap));
+      updatedChildren.map((child) => {
+        limitMaxNumberOfSeriesForPanel(child);
+      });
+
+      layout.setState({
+        children: updatedChildren,
+      });
+    });
+  }
+
+  private calculateCardinalityMap(detectedLabels?: DataFrame) {
+    const cardinalityMap = new Map<string, number>();
+    if (detectedLabels?.length) {
+      for (let i = 0; i < detectedLabels?.fields.length; i++) {
+        const name: string = detectedLabels.fields[i].name;
+        const cardinality: number = detectedLabels.fields[i].values[0];
+        cardinalityMap.set(name, cardinality);
+      }
+    }
+    return cardinalityMap;
+  }
+
   private build(): LayoutSwitcher {
     const variable = getLabelGroupByVariable(this);
     const labelBreakdownScene = sceneGraph.getAncestor(this, LabelBreakdownScene);
     labelBreakdownScene.state.search.reset();
-    const children: SceneCSSGridItem[] = [];
 
-    for (const option of variable.state.options) {
-      const { value } = option;
-      const optionValue = String(value);
-      if (value === ALL_VARIABLE_VALUE || !value) {
-        continue;
-      }
-      const query = buildLabelsQuery(this, String(option.value), String(option.value));
-      const dataTransformer = this.getDataTransformer(query);
+    const children = this.buildChildren(variable.state.options);
 
-      children.push(
-        new SceneCSSGridItem({
-          body: PanelBuilders.timeseries()
-            .setTitle(optionValue)
-            .setData(dataTransformer)
-            .setHeaderActions(new SelectLabelActionScene({ labelName: optionValue, fieldType: ValueSlugs.label }))
-            .setCustomFieldConfig('stacking', { mode: StackingMode.Normal })
-            .setCustomFieldConfig('fillOpacity', 100)
-            .setCustomFieldConfig('lineWidth', 0)
-            .setCustomFieldConfig('pointSize', 0)
-            .setCustomFieldConfig('drawStyle', DrawStyle.Bars)
-            .setOverrides(setLevelColorOverrides)
-            .build(),
-        })
-      );
+    const serviceScene = sceneGraph.getAncestor(this, ServiceScene);
+    const $detectedLabels = serviceScene.state.$detectedLabelsData;
+    if ($detectedLabels?.state.data?.state === LoadingState.Done) {
+      const cardinalityMap = this.calculateCardinalityMap($detectedLabels?.state.data.series[0]);
+      children.sort(this.sortChildren(cardinalityMap));
     }
 
     const childrenClones = children.map((child) => child.clone());
@@ -132,12 +205,58 @@ export class LabelsAggregatedBreakdownScene extends SceneObjectBase<LabelsAggreg
           isLazy: true,
           templateColumns: '1fr',
           autoRows: '200px',
-          children: children.map((child) => child.clone()),
+          children: childrenClones,
         }),
       ],
     });
   }
 
+  private buildChildren(options: VariableValueOption[]) {
+    const children: SceneCSSGridItem[] = [];
+    for (const option of options) {
+      const { value } = option;
+      const optionValue = String(value);
+      if (value === ALL_VARIABLE_VALUE || !value) {
+        continue;
+      }
+      const query = buildLabelsQuery(this, String(option.value), String(option.value));
+      const dataTransformer = this.getDataTransformer(query);
+
+      children.push(
+        new SceneCSSGridItem({
+          body: PanelBuilders.timeseries()
+            .setTitle(optionValue)
+            .setData(dataTransformer)
+            .setHeaderActions(new SelectLabelActionScene({ labelName: optionValue, fieldType: ValueSlugs.label }))
+            .setCustomFieldConfig('stacking', { mode: StackingMode.Normal })
+            .setCustomFieldConfig('fillOpacity', 100)
+            .setCustomFieldConfig('lineWidth', 0)
+            .setCustomFieldConfig('pointSize', 0)
+            .setCustomFieldConfig('drawStyle', DrawStyle.Bars)
+            .setOverrides(setLevelColorOverrides)
+            .build(),
+        })
+      );
+    }
+    return children;
+  }
+
+  private sortChildren(cardinalityMap: Map<string, number>) {
+    return (a: SceneCSSGridItem, b: SceneCSSGridItem) => {
+      const aPanel = a.state.body as VizPanel;
+      const bPanel = b.state.body as VizPanel;
+      if (aPanel.state.title === LEVEL_VARIABLE_VALUE) {
+        return -1;
+      }
+      if (bPanel.state.title === LEVEL_VARIABLE_VALUE) {
+        return 1;
+      }
+      const aCardinality = cardinalityMap.get(aPanel.state.title) ?? 0;
+      const bCardinality = cardinalityMap.get(bPanel.state.title) ?? 0;
+      return bCardinality - aCardinality;
+    };
+  }
+
   private getDataTransformer(query: LokiQuery) {
     const queryRunner = getQueryRunner([query]);
     return new SceneDataTransformer({

src/Components/ServiceScene/ServiceScene.tsx

@@ -25,6 +25,7 @@ import {
   getLabelsVariable,
   getLevelsVariable,
   getServiceNameFromVariableState,
+  LEVEL_VARIABLE_VALUE,
   LOG_STREAM_SELECTOR_EXPR,
   SERVICE_NAME,
   VAR_DATASOURCE,
@@ -260,10 +261,7 @@ export class ServiceScene extends SceneObjectBase<ServiceSceneState> {
     }
 
     // If we don't have a detected labels count, or we are activating the labels scene, run the detected labels query
-    if (
-      ((slug === PageSlugs.labels || parentSlug === ValueSlugs.label) && !this.state.$detectedLabelsData?.state.data) ||
-      this.state.labelsCount === undefined
-    ) {
+    if (slug === PageSlugs.labels || parentSlug === ValueSlugs.label || this.state.labelsCount === undefined) {
       this.state.$detectedLabelsData?.runQueries();
     }
 
@@ -297,8 +295,12 @@ export class ServiceScene extends SceneObjectBase<ServiceSceneState> {
         // Detected labels API call always returns a single frame, with a field for each label
         const detectedLabelsFields = detectedLabelsResponse.series[0].fields;
         if (detectedLabelsResponse.series.length !== undefined && detectedLabelsFields.length !== undefined) {
+          const removeSpecialFields = detectedLabelsResponse.series[0].fields.filter(
+            (f) => LEVEL_VARIABLE_VALUE !== f.name
+          );
+
           this.setState({
-            labelsCount: detectedLabelsFields.length,
+            labelsCount: removeSpecialFields.length + 1, // Add one for detected_level
           });
           getMetadataService().setLabelsCount(detectedLabelsFields.length);
         }

src/services/datasource.ts

@@ -16,8 +16,8 @@ import { getDataSource } from './scenes';
 import { LokiQuery } from './query';
 import { PLUGIN_ID } from './routing';
 import { DetectedFieldsResponse, DetectedLabelsResponse } from './fields';
-import { FIELDS_TO_REMOVE, sortLabelsByCardinality } from './filters';
-import { LEVEL_VARIABLE_VALUE, SERVICE_NAME } from './variables';
+import { FIELDS_TO_REMOVE, LABELS_TO_REMOVE, sortLabelsByCardinality } from './filters';
+import { SERVICE_NAME } from './variables';
 import { runShardSplitQuery } from './shardQuerySplitting';
 import { requestSupportsSharding } from './logql';
 
@@ -286,8 +286,8 @@ export class WrappedLokiDatasource extends RuntimeDataSource<DataQuery> {
         }
       );
       const labels = response.detectedLabels
-        ?.sort((a, b) => sortLabelsByCardinality(a, b))
-        ?.filter((label) => label.label !== LEVEL_VARIABLE_VALUE);
+        ?.filter((label) => !LABELS_TO_REMOVE.includes(label.label))
+        ?.sort((a, b) => sortLabelsByCardinality(a, b));
 
       const detectedLabelFields: Array<Partial<Field>> = labels?.map((label) => {
         return {

src/services/filters.test.ts

@@ -81,17 +81,6 @@ describe('getLabelOptions', () => {
     expect(getLabelOptions(labels)).toEqual(expectedOptions);
   });
 
-  it('should remove level if it is present in the list', () => {
-    const labels = [LEVEL_VARIABLE_VALUE, 'level', 'Label A'];
-    const expectedOptions: Array<SelectableValue<string>> = [
-      { label: 'All', value: ALL_VARIABLE_VALUE },
-      { label: LEVEL_VARIABLE_VALUE, value: LEVEL_VARIABLE_VALUE },
-      { label: 'Label A', value: 'Label A' },
-    ];
-
-    expect(getLabelOptions(labels)).toEqual(expectedOptions);
-  });
-
   it('should always add the All option at the beginning', () => {
     const labels = ['Label A', 'Label B'];
     const expectedOptions: Array<SelectableValue<string>> = [

src/services/filters.ts

@@ -1,5 +1,5 @@
 import { DetectedLabel, LabelType } from './fields';
-import { ALL_VARIABLE_VALUE, LEVEL_VARIABLE_VALUE } from './variables';
+import { ALL_VARIABLE_VALUE, LEVEL_VARIABLE_VALUE, SERVICE_NAME } from './variables';
 import { VariableValueOption } from '@grafana/scenes';
 
 export enum FilterOp {
@@ -33,10 +33,6 @@ export function getLabelOptions(labels: string[]) {
   if (!labels.includes(LEVEL_VARIABLE_VALUE)) {
     options.unshift(LEVEL_VARIABLE_VALUE);
   }
-  const labelsIndex = options.indexOf('level');
-  if (labelsIndex !== -1) {
-    options.splice(labelsIndex, 1);
-  }
 
   const labelOptions: VariableValueOption[] = options.map((label) => ({
     label,
@@ -45,7 +41,9 @@ export function getLabelOptions(labels: string[]) {
 
   return [{ label: 'All', value: ALL_VARIABLE_VALUE }, ...labelOptions];
 }
-export const FIELDS_TO_REMOVE = ['level_extracted', LEVEL_VARIABLE_VALUE, 'level'];
+export const LEVEL_INDEX_NAME = 'level';
+export const FIELDS_TO_REMOVE = ['level_extracted', LEVEL_VARIABLE_VALUE, LEVEL_INDEX_NAME];
+export const LABELS_TO_REMOVE = [SERVICE_NAME];
 
 export function getFieldOptions(labels: string[]) {
   const options = [...labels];