From 34a724d68ee25f108d5834b3c6797566153eac71 Mon Sep 17 00:00:00 2001 From: Ben Sully Date: Fri, 6 Sep 2024 15:49:43 +0100 Subject: [PATCH] feat: run augurs computation in worker thread This is mostly a PoC PR demonstrating how we might do this, in case it comes up in future. I've used workerize-loader to simplify the creation of the worker but we could probably do it by hand to avoid the dependency if we wanted. --- package.json | 3 +- .../Breakdowns/ByFrameRepeater.tsx | 4 +-- src/module.ts | 6 ---- src/services/augurs.ts | 4 +++ src/services/augursWorker.ts | 29 ++++++++++++++++ src/services/sorting.ts | 33 +++++++++++-------- webpack.config.ts | 3 ++ yarn.lock | 28 +++++++++++++++- 8 files changed, 87 insertions(+), 23 deletions(-) create mode 100644 src/services/augurs.ts create mode 100644 src/services/augursWorker.ts diff --git a/package.json b/package.json index 856b83376..18c687614 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,8 @@ "typescript": "4.8.4", "webpack": "^5.86.0", "webpack-cli": "^5.1.4", - "webpack-livereload-plugin": "^3.0.2" + "webpack-livereload-plugin": "^3.0.2", + "workerize-loader": "^2.0.2" }, "engines": { "node": ">=20" diff --git a/src/Components/ServiceScene/Breakdowns/ByFrameRepeater.tsx b/src/Components/ServiceScene/Breakdowns/ByFrameRepeater.tsx index 4c53fbc41..d5d53d7e9 100644 --- a/src/Components/ServiceScene/Breakdowns/ByFrameRepeater.tsx +++ b/src/Components/ServiceScene/Breakdowns/ByFrameRepeater.tsx @@ -71,9 +71,9 @@ export class ByFrameRepeater extends SceneObjectBase { } }; - private performRepeat(data: PanelData) { + private async performRepeat(data: PanelData) { const newChildren: SceneFlexItem[] = []; - const sortedSeries = sortSeries(data.series, this.sortBy, this.direction); + const sortedSeries = await sortSeries(data.series, this.sortBy, this.direction); for (let seriesIndex = 0; seriesIndex < sortedSeries.length; seriesIndex++) { const layoutChild = this.state.getLayoutChild(sortedSeries[seriesIndex], seriesIndex); diff --git a/src/module.ts b/src/module.ts index 3261ce89a..15e7d4a6f 100644 --- a/src/module.ts +++ b/src/module.ts @@ -1,13 +1,7 @@ import { AppPlugin } from '@grafana/data'; import { App } from 'Components/App'; -import init from '@bsull/augurs'; import { linkConfigs } from 'services/extensions/links'; import { init as initRuntimeDs } from 'services/datasource'; -import { wasmSupported } from 'services/sorting'; - -if (wasmSupported()) { - init(); -} export const plugin = new AppPlugin<{}>().setRootPage(App); diff --git a/src/services/augurs.ts b/src/services/augurs.ts new file mode 100644 index 000000000..66e0e8578 --- /dev/null +++ b/src/services/augurs.ts @@ -0,0 +1,4 @@ +// @ts-ignore no-default-export +import worker from 'workerize-loader?name=augurs&ready!./augursWorker'; + +export const augurs = worker(); diff --git a/src/services/augursWorker.ts b/src/services/augursWorker.ts new file mode 100644 index 000000000..bfb8f640a --- /dev/null +++ b/src/services/augursWorker.ts @@ -0,0 +1,29 @@ +import init, { type Changepoints, ChangepointDetector, OutlierDetector, OutlierOutput } from '@bsull/augurs'; + +const wasmSupported = () => { + const support = typeof WebAssembly === 'object'; + + // if (!support) { + // reportAppInteraction(USER_EVENTS_PAGES.service_details, USER_EVENTS_ACTIONS.service_details.wasm_not_supported); + // } + + return support; +}; + +if (wasmSupported()) { + init(); +} + +export function detectChangepoints(values: Float64Array): Changepoints { + return ChangepointDetector.defaultArgpcp().detectChangepoints(values); +} + +interface DetectOutliersOptions { + points: Float64Array; + nTimestamps: number; + sensitivity: number; +} + +export function detectOutliers(opts: DetectOutliersOptions): OutlierOutput { + return OutlierDetector.dbscan({ sensitivity: opts.sensitivity }).preprocess(opts.points, opts.nTimestamps).detect(); +} diff --git a/src/services/sorting.ts b/src/services/sorting.ts index 1cca42251..24818e85e 100644 --- a/src/services/sorting.ts +++ b/src/services/sorting.ts @@ -1,22 +1,24 @@ -import { ChangepointDetector, OutlierDetector, OutlierOutput } from '@bsull/augurs'; +import { type OutlierOutput } from '@bsull/augurs'; import { DataFrame, FieldType, ReducerID, doStandardCalcs, fieldReducers, outerJoinDataFrames } from '@grafana/data'; import { getLabelValueFromDataFrame } from './levels'; import { memoize } from 'lodash'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from './analytics'; +import { augurs } from './augurs'; + export const DEFAULT_SORT_BY = 'changepoint'; export const sortSeries = memoize( - (series: DataFrame[], sortBy: string, direction: string) => { + async (series: DataFrame[], sortBy: string, direction: string) => { if (sortBy === 'alphabetical') { return sortSeriesByName(series, direction); } if (sortBy === 'outliers') { - initOutlierDetector(series); + await initOutlierDetector(series); } - const reducer = (dataFrame: DataFrame) => { + const reducer = async (dataFrame: DataFrame) => { // ML & Wasm sorting options try { if (sortBy === 'changepoint') { @@ -35,10 +37,12 @@ export const sortSeries = memoize( return value[sortBy] ?? 0; }; - const seriesCalcs = series.map((dataFrame) => ({ - value: reducer(dataFrame), - dataFrame: dataFrame, - })); + const seriesCalcs = await Promise.all( + series.map(async (dataFrame) => ({ + value: await reducer(dataFrame), + dataFrame: dataFrame, + })) + ); seriesCalcs.sort((a, b) => { if (a.value !== undefined && b.value !== undefined) { @@ -66,7 +70,7 @@ export const sortSeries = memoize( } ); -export const calculateDataFrameChangepoints = (data: DataFrame) => { +export const calculateDataFrameChangepoints = async (data: DataFrame) => { if (!wasmSupported()) { throw new Error('WASM not supported, fall back to stdDev'); } @@ -84,7 +88,7 @@ export const calculateDataFrameChangepoints = (data: DataFrame) => { const sample = fields[0].values.filter((_, i) => i % samplingStep === 0); const values = new Float64Array(sample); - const points = ChangepointDetector.defaultArgpcp().detectChangepoints(values); + const points = await augurs.detectChangepoints(values); return points.indices.length; }; @@ -105,7 +109,7 @@ export const sortSeriesByName = (series: DataFrame[], direction: string) => { return sortedSeries; }; -const initOutlierDetector = (series: DataFrame[]) => { +const initOutlierDetector = async (series: DataFrame[]) => { if (!wasmSupported()) { return; } @@ -122,8 +126,11 @@ const initOutlierDetector = (series: DataFrame[]) => { const points = new Float64Array(joinedSeries.flatMap((series) => series.values as number[])); try { - const detector = OutlierDetector.dbscan({ sensitivity: 0.4 }).preprocess(points, nTimestamps); - outliers = detector.detect(); + outliers = await augurs.detectOutliers({ + sensitivity: 0.4, + points, + nTimestamps, + }); } catch (e) { console.error(e); } diff --git a/webpack.config.ts b/webpack.config.ts index 623b503cc..2c75294b9 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -9,6 +9,9 @@ const config = async (env): Promise => { // Required to load WASM modules. asyncWebAssembly: true, }, + output: { + publicPath: '/public/plugins/grafana-lokiexplore-app/', + }, }); }; diff --git a/yarn.lock b/yarn.lock index da686085e..a78bedd51 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3005,6 +3005,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +big.js@^5.2.2: + version "5.2.2" + resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" + integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== + binary-extensions@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" @@ -4149,6 +4154,11 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== +emojis-list@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78" + integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q== + enhanced-resolve@^5.16.0: version "5.16.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.16.0.tgz#65ec88778083056cb32487faa9aef82ed0864787" @@ -6160,7 +6170,7 @@ json-stringify-pretty-compact@^2.0.0: resolved "https://registry.yarnpkg.com/json-stringify-pretty-compact/-/json-stringify-pretty-compact-2.0.0.tgz#e77c419f52ff00c45a31f07f4c820c2433143885" integrity sha512-WRitRfs6BGq4q8gTgOy4ek7iPFXjbra0H3PmDLKm2xnZ+Gh1HUhiKGgCZkSPNULlP7mvfu6FV/mOLhCarspADQ== -json5@^2.2.2, json5@^2.2.3: +json5@^2.1.2, json5@^2.2.2, json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== @@ -6249,6 +6259,15 @@ loader-runner@^4.2.0: resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1" integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg== +loader-utils@^2.0.0: + version "2.0.4" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.4.tgz#8b5cb38b5c34a9a018ee1fc0e6a066d1dfcc528c" + integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw== + dependencies: + big.js "^5.2.2" + emojis-list "^3.0.0" + json5 "^2.1.2" + locate-path@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" @@ -9080,6 +9099,13 @@ wildcard@^2.0.0: resolved "https://registry.yarnpkg.com/wildcard/-/wildcard-2.0.1.tgz#5ab10d02487198954836b6349f74fff961e10f67" integrity sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ== +workerize-loader@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/workerize-loader/-/workerize-loader-2.0.2.tgz#2f39c3e6eb6c41fac8d28d5b9fa20b4035e3db7a" + integrity sha512-HoZ6XY4sHWxA2w0WpzgBwUiR3dv1oo7bS+oCwIpb6n54MclQ/7KXdXsVIChTCygyuHtVuGBO1+i3HzTt699UJQ== + dependencies: + loader-utils "^2.0.0" + "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"