From 7fc099d08130a83edffcf7f75a29bc1f8087ad00 Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Thu, 19 Sep 2024 08:11:13 -0500 Subject: [PATCH] ServiceSelection: add support for aggregated metric (#713) * feat: add experimental support for `exploreLogsAggregatedMetrics` in service selection --------- Co-authored-by: Sven Grossmann --- package.json | 2 +- src/Components/IndexScene/IndexScene.tsx | 27 ++- src/Components/IndexScene/ToolbarScene.tsx | 140 +++++++++++++++ .../ServiceSelectionScene.tsx | 161 ++++++++++++++++-- src/services/analytics.ts | 2 + src/services/datasource.ts | 57 ++++++- src/services/panel.ts | 15 +- src/services/query.ts | 2 +- src/services/variables.ts | 10 ++ tests/exploreServices.spec.ts | 5 +- tests/mocks/mockLabelsResponse.ts | 4 + yarn.lock | 8 +- 12 files changed, 395 insertions(+), 38 deletions(-) create mode 100644 src/Components/IndexScene/ToolbarScene.tsx create mode 100644 tests/mocks/mockLabelsResponse.ts diff --git a/package.json b/package.json index 5a9fcc8df..55fd15bad 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "@grafana/data": "^11.1.1", "@grafana/lezer-logql": "^0.2.6", "@grafana/runtime": "^11.1.1", - "@grafana/scenes": "5.7.2", + "@grafana/scenes": "5.9.0", "@grafana/ui": "^11.1.1", "@lezer/common": "^1.2.1", "@lezer/lr": "^1.4.1", diff --git a/src/Components/IndexScene/IndexScene.tsx b/src/Components/IndexScene/IndexScene.tsx index a32b6fdd2..afb42978d 100644 --- a/src/Components/IndexScene/IndexScene.tsx +++ b/src/Components/IndexScene/IndexScene.tsx @@ -43,7 +43,7 @@ import { FilterOp } from 'services/filters'; import { getDrilldownSlug, PageSlugs } from '../../services/routing'; import { ServiceSelectionScene } from '../ServiceSelectionScene/ServiceSelectionScene'; import { LoadingPlaceholder } from '@grafana/ui'; -import { locationService } from '@grafana/runtime'; +import { config, locationService } from '@grafana/runtime'; import { renderLogQLFieldFilters, renderLogQLLabelFilters, @@ -52,6 +52,7 @@ import { } from 'services/query'; import { VariableHide } from '@grafana/schema'; import { CustomConstantVariable } from '../../services/CustomConstantVariable'; +import { ToolbarScene } from './ToolbarScene'; export interface AppliedPattern { pattern: string; @@ -76,15 +77,27 @@ export class IndexScene extends SceneObjectBase { getLastUsedDataSourceFromStorage() ?? 'grafanacloud-logs', state.initialFilters ); + + const controls: SceneObject[] = [ + new VariableValueSelectors({ layout: 'vertical' }), + new SceneControlsSpacer(), + new SceneTimePicker({}), + new SceneRefreshPicker({}), + ]; + + //@ts-expect-error + if (getDrilldownSlug() === 'explore' && config.featureToggles.exploreLogsAggregatedMetrics) { + controls.push( + new ToolbarScene({ + isOpen: false, + }) + ); + } + super({ $timeRange: state.$timeRange ?? new SceneTimeRange({}), $variables: state.$variables ?? variablesScene, - controls: state.controls ?? [ - new VariableValueSelectors({ layout: 'vertical' }), - new SceneControlsSpacer(), - new SceneTimePicker({}), - new SceneRefreshPicker({}), - ], + controls: state.controls ?? controls, // Need to clear patterns state when the class in constructed patterns: [], ...state, diff --git a/src/Components/IndexScene/ToolbarScene.tsx b/src/Components/IndexScene/ToolbarScene.tsx new file mode 100644 index 000000000..ce1b0598d --- /dev/null +++ b/src/Components/IndexScene/ToolbarScene.tsx @@ -0,0 +1,140 @@ +import { SceneComponentProps, SceneObjectBase, SceneObjectState } from '@grafana/scenes'; +import { Dropdown, Switch, ToolbarButton, useStyles2 } from '@grafana/ui'; +import React from 'react'; +import { GrafanaTheme2 } from '@grafana/data'; +import { css } from '@emotion/css'; +import { config } from '@grafana/runtime'; +import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '../../services/analytics'; +import { AGGREGATED_METRIC_START_DATE } from '../ServiceSelectionScene/ServiceSelectionScene'; +import pluginJson from '../../plugin.json'; +const AGGREGATED_METRICS_USER_OVERRIDE_LOCALSTORAGE_KEY = `${pluginJson.id}.serviceSelection.aggregatedMetrics`; + +export interface ToolbarSceneState extends SceneObjectState { + isOpen: boolean; + options: { + aggregatedMetrics: { + active: boolean; + userOverride: boolean; + disabled: boolean; + }; + }; +} +export class ToolbarScene extends SceneObjectBase { + constructor(state: Partial) { + const userOverride = localStorage.getItem(AGGREGATED_METRICS_USER_OVERRIDE_LOCALSTORAGE_KEY); + // @ts-expect-error + const active = config.featureToggles.exploreLogsAggregatedMetrics && userOverride !== 'false'; + + super({ + isOpen: false, + options: { + aggregatedMetrics: { + active, + userOverride: userOverride === 'true' ?? false, + disabled: false, + }, + }, + ...state, + }); + } + + public toggleAggregatedMetricsOverride = () => { + const active = !this.state.options.aggregatedMetrics.active; + + reportAppInteraction( + USER_EVENTS_PAGES.service_selection, + USER_EVENTS_ACTIONS.service_selection.aggregated_metrics_toggled, + { + enabled: active, + } + ); + + localStorage.setItem(AGGREGATED_METRICS_USER_OVERRIDE_LOCALSTORAGE_KEY, active.toString()); + + this.setState({ + options: { + aggregatedMetrics: { + active, + disabled: this.state.options.aggregatedMetrics.disabled, + userOverride: active, + }, + }, + }); + }; + + public onToggleOpen = (isOpen: boolean) => { + this.setState({ isOpen }); + }; + + static Component = ({ model }: SceneComponentProps) => { + const { isOpen, options } = model.useState(); + const styles = useStyles2(getStyles); + + const renderPopover = () => { + return ( +
evt.stopPropagation()}> +
Query options
+
+
+ Aggregated metrics +
+ + + +
+
+ ); + }; + + if (options.aggregatedMetrics) { + return ( + + + + ); + } + + return <>; + }; +} + +function getStyles(theme: GrafanaTheme2) { + return { + popover: css({ + display: 'flex', + padding: theme.spacing(2), + flexDirection: 'column', + background: theme.colors.background.primary, + boxShadow: theme.shadows.z3, + borderRadius: theme.shape.radius.default, + border: `1px solid ${theme.colors.border.weak}`, + zIndex: 1, + marginRight: theme.spacing(2), + }), + heading: css({ + fontWeight: theme.typography.fontWeightMedium, + paddingBottom: theme.spacing(2), + }), + options: css({ + display: 'grid', + gridTemplateColumns: '1fr 50px', + rowGap: theme.spacing(1), + columnGap: theme.spacing(2), + alignItems: 'center', + }), + }; +} diff --git a/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx b/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx index fc183b9a3..9230b87da 100644 --- a/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx +++ b/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx @@ -1,17 +1,17 @@ import { css } from '@emotion/css'; import { debounce } from 'lodash'; import React from 'react'; -import { DashboardCursorSync, DataFrame, GrafanaTheme2, LoadingState, TimeRange, VariableHide } from '@grafana/data'; +import { DashboardCursorSync, DataFrame, dateTime, GrafanaTheme2, LoadingState, TimeRange } from '@grafana/data'; import { behaviors, PanelBuilders, SceneComponentProps, SceneCSSGridItem, SceneCSSGridLayout, - SceneDataProvider, sceneGraph, SceneObjectBase, SceneObjectState, + SceneQueryRunner, SceneVariableSet, VizPanel, } from '@grafana/scenes'; @@ -29,23 +29,37 @@ import { getFavoriteServicesFromStorage } from 'services/store'; import { getDataSourceVariable, getLabelsVariable, + getServiceNameVariable, getServiceSelectionStringVariable, LEVEL_VARIABLE_VALUE, SERVICE_NAME, + SERVICE_NAME_EXPR, + SERVICE_NAME_VAR, VAR_SERVICE, VAR_SERVICE_EXPR, } from 'services/variables'; import { selectService, SelectServiceButton } from './SelectServiceButton'; import { buildDataQuery, buildResourceQuery } from 'services/query'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; -import { getQueryRunner, setLevelColorOverrides } from 'services/panel'; +import { getQueryRunner, getSceneQueryRunner, setLevelColorOverrides } from 'services/panel'; import { ConfigureVolumeError } from './ConfigureVolumeError'; import { NoVolumeError } from './NoVolumeError'; import { getLabelsFromSeries, toggleLevelVisibility } from 'services/levels'; import { ServiceFieldSelector } from '../ServiceScene/Breakdowns/FieldSelector'; import { CustomConstantVariable } from '../../services/CustomConstantVariable'; import { areArraysEqual } from '../../services/comparison'; - +import { config } from '@grafana/runtime'; +import { VariableHide } from '@grafana/schema'; +import { ToolbarScene } from '../IndexScene/ToolbarScene'; +import { IndexScene } from '../IndexScene/IndexScene'; + +// @ts-expect-error +const aggregatedMetricsEnabled: boolean | undefined = config.featureToggles.exploreLogsAggregatedMetrics; +// Don't export AGGREGATED_SERVICE_NAME, we want to rename things so the rest of the application is agnostic to how we got the services +const AGGREGATED_SERVICE_NAME = '__aggregated_metric__'; + +//@todo make start date user configurable, currently hardcoded for experimental cloud release +export const AGGREGATED_METRIC_START_DATE = dateTime('2024-08-30', 'YYYY-MM-DD'); export const SERVICES_LIMIT = 20; interface ServiceSelectionSceneState extends SceneObjectState { @@ -54,15 +68,7 @@ interface ServiceSelectionSceneState extends SceneObjectState { // Show logs of a certain level for a given service serviceLevel: Map; // Logs volume API response as dataframe with SceneQueryRunner - $data: SceneDataProvider; -} - -function getMetricExpression(service: string) { - return `sum by (${LEVEL_VARIABLE_VALUE}) (count_over_time({${SERVICE_NAME}=\`${service}\`} | drop __error__ [$__auto]))`; -} - -function getLogExpression(service: string, levelFilter: string) { - return `{${SERVICE_NAME}=\`${service}\`}${levelFilter}`; + $data: SceneQueryRunner; } export class ServiceSelectionScene extends SceneObjectBase { @@ -71,6 +77,7 @@ export class ServiceSelectionScene extends SceneObjectBase(), ...state, }); @@ -108,6 +136,96 @@ export class ServiceSelectionScene extends SceneObjectBase { + if (this.isTimeRangeTooEarlyForAggMetrics()) { + this.onUnsupportedAggregatedMetricTimeRange(); + } else { + this.onSupportedAggregatedMetricTimeRange(); + } + this.runServiceQueries(); + }) + ); + + // Update labels on datasource change + this._subs.add( + getDataSourceVariable(this).subscribeToState(() => { + this.runServiceQueries(); + }) + ); + + if (aggregatedMetricsEnabled) { + this._subs.add( + this.getQueryOptionsToolbar()?.subscribeToState((newState, prevState) => { + if (newState.options.aggregatedMetrics.userOverride !== prevState.options.aggregatedMetrics.userOverride) { + this.runServiceQueries(); + } + }) + ); + } + } + + private isTimeRangeTooEarlyForAggMetrics(): boolean { + const timeRange = sceneGraph.getTimeRange(this); + return timeRange.state.value.from.isBefore(dateTime(AGGREGATED_METRIC_START_DATE)); + } + + private onUnsupportedAggregatedMetricTimeRange() { + const toolbar = this.getQueryOptionsToolbar(); + toolbar?.setState({ + options: { + aggregatedMetrics: { + ...toolbar?.state.options.aggregatedMetrics, + disabled: true, + }, + }, + }); + } + + private getQueryOptionsToolbar() { + const indexScene = sceneGraph.getAncestor(this, IndexScene); + return indexScene.state.controls.find((control) => control instanceof ToolbarScene) as ToolbarScene | undefined; + } + + private onSupportedAggregatedMetricTimeRange() { + const toolbar = this.getQueryOptionsToolbar(); + toolbar?.setState({ + options: { + aggregatedMetrics: { + ...toolbar?.state.options.aggregatedMetrics, + disabled: false, + }, + }, + }); + } + + private runServiceQueries() { + const toolbar = this.getQueryOptionsToolbar(); + const aggregatedMetricsActive = + !toolbar?.state.options.aggregatedMetrics.disabled && toolbar?.state.options.aggregatedMetrics.active; + + if ((!this.isTimeRangeTooEarlyForAggMetrics() || !aggregatedMetricsEnabled) && aggregatedMetricsActive) { + const serviceName = getServiceNameVariable(this); + serviceName.changeValueTo(AGGREGATED_SERVICE_NAME); + } else { + const serviceName = getServiceNameVariable(this); + serviceName.changeValueTo(SERVICE_NAME); + } + this.state.$data.runQueries(); } private updateBody() { @@ -170,6 +288,14 @@ export class ServiceSelectionScene extends SceneObjectBase { const originalOnToggleSeriesVisibility = context.onToggleSeriesVisibility; @@ -195,7 +321,7 @@ export class ServiceSelectionScene extends SceneObjectBase { const levelFilter = this.getLevelFilterForService(service); - // const timeRange = sceneGraph.getTimeRange(this).state.value; return new SceneCSSGridItem({ $behaviors: [new behaviors.CursorSync({ sync: DashboardCursorSync.Off })], body: PanelBuilders.logs() @@ -253,10 +378,9 @@ export class ServiceSelectionScene extends SceneObjectBase { const variable = getServiceSelectionStringVariable(this); variable.changeValueTo(serviceString ?? ''); + this.state.$data.runQueries(); reportAppInteraction( USER_EVENTS_PAGES.service_selection, diff --git a/src/services/analytics.ts b/src/services/analytics.ts index 17f7e5c25..93271411b 100644 --- a/src/services/analytics.ts +++ b/src/services/analytics.ts @@ -31,6 +31,8 @@ export const USER_EVENTS_ACTIONS = { search_services_changed: 'search_services_changed', // Selecting service. Props: service service_selected: 'service_selected', + // Toggling aggregated metrics on/off + aggregated_metrics_toggled: 'aggregated_metrics_toggled', }, [USER_EVENTS_PAGES.service_details]: { open_in_explore_clicked: 'open_in_explore_clicked', diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 81ddb61a1..6909afc31 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -28,14 +28,15 @@ export type SceneDataQueryRequest = DataQueryRequest { await this.getDetectedFields(request, ds, subscriber); break; } + case 'labels': { + await this.getLabels(request, ds, subscriber); + break; + } default: { this.getData(request, ds, subscriber); break; @@ -413,7 +423,10 @@ export class WrappedLokiDatasource extends RuntimeDataSource { // Scenes will only emit dataframes from the SceneQueryRunner, so for now we need to convert the API response to a dataframe const df = createDataFrame({ fields: [ - { name: SERVICE_NAME, values: volumeResponse?.data.result?.map((r) => r.metric.service_name) }, + { + name: SERVICE_NAME, + values: volumeResponse?.data.result?.map((r) => r.metric.service_name ?? r.metric.__aggregated_metric__), + }, { name: 'volume', values: volumeResponse?.data.result?.map((r) => Number(r.value[1])) }, ], }); @@ -427,6 +440,44 @@ export class WrappedLokiDatasource extends RuntimeDataSource { return subscriber; } + private async getLabels( + request: DataQueryRequest, + ds: DataSourceWithBackend, + subscriber: Subscriber + ) { + if (request.targets.length !== 1) { + throw new Error('Volume query can only have a single target!'); + } + + try { + const labelsResponse: LabelsResponse = await ds.getResource( + 'labels', + { + start: request.range.from.utc().toISOString(), + end: request.range.to.utc().toISOString(), + }, + { + requestId: request.requestId ?? 'labels', + headers: { + 'X-Query-Tags': `Source=${PLUGIN_ID}`, + }, + } + ); + + // Scenes will only emit dataframes from the SceneQueryRunner, so for now we need to convert the API response to a dataframe + const df = createDataFrame({ + fields: [{ name: 'labels', values: labelsResponse?.data }], + }); + subscriber.next({ data: [df], state: LoadingState.Done }); + } catch (e) { + subscriber.next({ data: [], state: LoadingState.Error }); + } + + subscriber.complete(); + + return subscriber; + } + testDatasource(): Promise { return Promise.resolve({ status: 'success', message: 'Data source is working', title: 'Success' }); } diff --git a/src/services/panel.ts b/src/services/panel.ts index fea83adfb..d1b54dacc 100644 --- a/src/services/panel.ts +++ b/src/services/panel.ts @@ -4,6 +4,7 @@ import { FieldConfigBuilders, FieldConfigOverridesBuilder, PanelBuilders, + QueryRunnerState, SceneDataProvider, SceneDataTransformer, SceneObject, @@ -123,7 +124,7 @@ export function getResourceQueryRunner(queries: LokiQuery[]) { }); } -export function getQueryRunner(queries: LokiQuery[]) { +export function getQueryRunner(queries: LokiQuery[], queryRunnerOptions?: Partial) { // if there's a legendFormat related to any `level` like label, we want to // sort the output equally. That's purposefully not `LEVEL_VARIABLE_VALUE`, // such that the `detected_level` graph looks the same as a graph for the @@ -133,7 +134,7 @@ export function getQueryRunner(queries: LokiQuery[]) { if (hasLevel) { return new SceneDataTransformer({ - $data: new SceneQueryRunner({ + $data: getSceneQueryRunner({ datasource: { uid: WRAPPED_LOKI_DS_UID }, queries: queries, }), @@ -141,9 +142,17 @@ export function getQueryRunner(queries: LokiQuery[]) { }); } + return getSceneQueryRunner({ + queries: queries, + ...queryRunnerOptions, + }); +} + +export function getSceneQueryRunner(queryRunnerOptions?: Partial) { return new SceneQueryRunner({ datasource: { uid: WRAPPED_LOKI_DS_UID }, - queries: queries, + queries: [], + ...queryRunnerOptions, }); } diff --git a/src/services/query.ts b/src/services/query.ts index ce59b789b..e591f9223 100644 --- a/src/services/query.ts +++ b/src/services/query.ts @@ -27,7 +27,7 @@ export type LokiQuery = { */ export const buildResourceQuery = ( expr: string, - resource: 'volume' | 'patterns' | 'detected_labels' | 'detected_fields', + resource: 'volume' | 'patterns' | 'detected_labels' | 'detected_fields' | 'labels', queryParamsOverrides?: Record ): LokiQuery & SceneDataQueryResourceRequest => { return { diff --git a/src/services/variables.ts b/src/services/variables.ts index eb6a9263a..c83cd63e7 100644 --- a/src/services/variables.ts +++ b/src/services/variables.ts @@ -39,6 +39,8 @@ export const EXPLORATION_DS = { uid: VAR_DATASOURCE_EXPR }; export const ALL_VARIABLE_VALUE = '$__all'; export const LEVEL_VARIABLE_VALUE = 'detected_level'; export const SERVICE_NAME = 'service_name'; +export const SERVICE_NAME_VAR = 'service_name_var'; +export const SERVICE_NAME_EXPR = '${service_name_var}'; export const EMPTY_VARIABLE_VALUE = '""'; export type ParserType = 'logfmt' | 'json' | 'mixed' | 'structuredMetadata'; @@ -106,6 +108,14 @@ export function getLabelGroupByVariable(scene: SceneObject) { return variable; } +export function getServiceNameVariable(scene: SceneObject) { + const variable = sceneGraph.lookupVariable(SERVICE_NAME_VAR, scene); + if (!(variable instanceof CustomConstantVariable)) { + throw new Error('SERVICE_NAME_VAR not found'); + } + return variable; +} + export function getFieldGroupByVariable(scene: SceneObject) { const variable = sceneGraph.lookupVariable(VAR_FIELD_GROUP_BY, scene); if (!(variable instanceof CustomConstantVariable)) { diff --git a/tests/exploreServices.spec.ts b/tests/exploreServices.spec.ts index 82e9e973b..aae00a866 100644 --- a/tests/exploreServices.spec.ts +++ b/tests/exploreServices.spec.ts @@ -2,6 +2,7 @@ import { test, expect } from '@grafana/plugin-e2e'; import { ExplorePage } from './fixtures/explore'; import { testIds } from '../src/services/testIds'; import { mockVolumeApiResponse } from './mocks/mockVolumeApiResponse'; +import { mockLabelsResponse } from './mocks/mockLabelsResponse'; test.describe('explore services page', () => { let explorePage: ExplorePage; @@ -125,7 +126,7 @@ test.describe('explore services page', () => { }); test.describe('mock volume API calls', () => { - let logsVolumeCount: number, logsQueryCount: number; + let logsVolumeCount: number, logsQueryCount: number, labelsQueryCount: number; test.beforeEach(async ({ page }) => { logsVolumeCount = 0; @@ -194,6 +195,7 @@ test.describe('explore services page', () => { test('changing datasource will trigger new queries', async ({ page }) => { await page.waitForFunction(() => !document.querySelector('[title="Cancel query"]')); + await explorePage.assertPanelsNotLoading(); expect(logsVolumeCount).toEqual(1); expect(logsQueryCount).toEqual(4); await page @@ -202,6 +204,7 @@ test.describe('explore services page', () => { .nth(1) .click(); await page.getByText('gdev-loki-copy').click(); + await explorePage.assertPanelsNotLoading(); expect(logsVolumeCount).toEqual(2); }); }); diff --git a/tests/mocks/mockLabelsResponse.ts b/tests/mocks/mockLabelsResponse.ts new file mode 100644 index 000000000..59fcfafec --- /dev/null +++ b/tests/mocks/mockLabelsResponse.ts @@ -0,0 +1,4 @@ +export const mockLabelsResponse = { + status: 'success', + data: ['__aggregated_metric__', '__stream_shard__', 'cluster', 'env', 'level', 'namespace', 'service_name'], +}; diff --git a/yarn.lock b/yarn.lock index 6cd2f72be..2564bd64a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1118,10 +1118,10 @@ rxjs "7.8.1" tslib "2.6.3" -"@grafana/scenes@5.7.2": - version "5.7.2" - resolved "https://registry.yarnpkg.com/@grafana/scenes/-/scenes-5.7.2.tgz#35b927cf71c118436bd8169c2ac095b16b44bc78" - integrity sha512-kKvUX6F23Btq072FfnFjlcDBST2/ejGMVjPe+1kR2JoMFRfepzYTbzV9N5EwXmwMvBSek4nE/DiWdV/L7sygBg== +"@grafana/scenes@5.9.0": + version "5.9.0" + resolved "https://registry.yarnpkg.com/@grafana/scenes/-/scenes-5.9.0.tgz#98dfc187a39ea4fa7d13ad91704514cddfddda5f" + integrity sha512-LP28gBAORNvCkeHPrxWT8Iz4W8Q5zBVDtbWwJ5YqzLucgSTJZWlYErfSS9+CSoeHx1imtfnF+mDBPj1anZzfTw== dependencies: "@grafana/e2e-selectors" "^11.0.0" "@leeoniya/ufuzzy" "^1.0.14"