Skip to content

Optimize LogField performance #585

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jwbonner merged 1 commit into from
Nov 12, 2026
Merged

Optimize LogField performance #585

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/hub/controllers/LineGraphController.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
LineGraphRendererCommand_NumericField
} from "../../shared/renderers/LineGraphRenderer";
import { Units } from "../../shared/units";
import { clampValue, createUUID, scaleValueClamped } from "../../shared/util";
import { clampValue, createUUID, lowerBound, scaleValueClamped, upperBound } from "../../shared/util";
import SourceList from "../SourceList";
import { LineGraphController_DiscreteConfig, LineGraphController_NumericConfig } from "./LineGraphController_Config";
import TabController from "./TabController";
Expand Down Expand Up @@ -352,7 +352,7 @@ export default class LineGraphController implements TabController {
return null;
}
let command = commandCache[key];
let index = command.timestamps.findLastIndex((sample) => sample <= time);
let index = upperBound(command.timestamps, time) - 1;
if (index === -1) {
return null;
}
Expand Down Expand Up @@ -480,7 +480,7 @@ export default class LineGraphController implements TabController {
if (fieldItem.type !== "stepped" || filter !== LineGraphFilter.None) {
let newData: LogValueSetNumber = { timestamps: [], values: [] };
let sourceIndex = 0;
let akitIndex = akitTimestamps.findIndex((akitTime) => akitTime >= data!.timestamps[0]);
let akitIndex = lowerBound(akitTimestamps, data!.timestamps[0]);
while (
akitIndex < akitTimestamps.length &&
akitTimestamps[akitIndex] <= data!.timestamps[data!.timestamps.length - 1]
Expand Down
18 changes: 9 additions & 9 deletions src/hub/exportWorker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { filterFieldByPrefixes } from "../shared/log/LogKeyUtils";
import { getLogValueText } from "../shared/log/LogUtil";
import LoggableType from "../shared/log/LoggableType";
import { AKIT_TIMESTAMP_KEYS } from "../shared/log/RobotState";
import { cleanFloat } from "../shared/util";
import { cleanFloat, upperBound } from "../shared/util";
import { WPILOGEncoder, WPILOGEncoderRecord } from "./dataSources/wpilog/WPILOGEncoder";

self.onmessage = async (event) => {
Expand Down Expand Up @@ -128,10 +128,10 @@ function generateCsvTable(
let fieldData = log.getRange(field, -Infinity, Infinity);
let fieldType = log.getType(field);

let nextIndex = 0;
timestamps!.forEach((timestamp, timestampIndex) => {
if (fieldData === undefined || fieldType === null) return;
let nextIndex = fieldData.timestamps.findIndex((value) => value > timestamp);
if (nextIndex === -1) nextIndex = fieldData.timestamps.length;
nextIndex = upperBound(fieldData.timestamps, timestamp, nextIndex);
let value: any = null;
if (nextIndex !== 0) {
value = fieldData.values[nextIndex - 1];
Expand Down Expand Up @@ -174,10 +174,10 @@ function generateCsvList(
});
} else {
// Add samples at timestamps
let nextIndex = 0;
timestamps.forEach((timestamp, timestampIndex) => {
if (fieldData === undefined) return;
let nextIndex = fieldData.timestamps.findIndex((value) => value > timestamp);
if (nextIndex === -1) nextIndex = fieldData.timestamps.length;
nextIndex = upperBound(fieldData.timestamps, timestamp, nextIndex);
let value: any = null;
if (nextIndex !== 0) {
value = fieldData.values[nextIndex - 1];
Expand Down Expand Up @@ -315,10 +315,10 @@ function generateWPILOG(
});
} else {
// Add samples at timestamps
let nextIndex = 0;
timestamps.forEach((timestamp, timestampIndex) => {
if (fieldData === undefined) return;
let nextIndex = fieldData.timestamps.findIndex((value) => value > timestamp);
if (nextIndex === -1) nextIndex = fieldData.timestamps.length;
nextIndex = upperBound(fieldData.timestamps, timestamp, nextIndex);
if (nextIndex !== 0) {
addValue(timestamp, fieldData.values[nextIndex - 1]);
}
Expand Down Expand Up @@ -411,10 +411,10 @@ async function generateMCAP(
progress((fieldIndex + i / fieldData.values.length) / filteredFields.length);
}
} else {
let nextIndex = 0;
for (let i = 0; i < timestamps.length; i++) {
let timestamp = timestamps[i];
let nextIndex = fieldData.timestamps.findIndex((value) => value > timestamp);
if (nextIndex === -1) nextIndex = fieldData.timestamps.length;
nextIndex = upperBound(fieldData.timestamps, timestamp, nextIndex);
if (nextIndex !== 0) {
addValue(timestamp, fieldData.values[nextIndex - 1], i);
}
Expand Down
15 changes: 9 additions & 6 deletions src/shared/log/Log.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import { Decoder } from "@msgpack/msgpack";
import { Pose2d, Translation2d } from "../geometry";
import { Units } from "../units";
import { arraysEqual, checkArrayType } from "../util";
import { arraysEqual, checkArrayType, lowerBound } from "../util";
import LogField from "./LogField";
import LogFieldTree from "./LogFieldTree";
import { PHOTON_PREFIX, STRUCT_PREFIX, applyKeyPrefix, getSchemaType, splitLogKey } from "./LogKeyUtils";
Expand Down Expand Up @@ -159,12 +159,15 @@ export default class Log {
// Update timestamp set caches
if (this.enableTimestampSetCache) {
Object.values(this.timestampSetCache).forEach((cache) => {
if (cache.keys.includes(key) && !cache.timestamps.includes(timestamp)) {
let insertIndex = cache.timestamps.findIndex((x) => x > timestamp);
if (insertIndex === -1) {
insertIndex = cache.timestamps.length;
if (cache.keys.includes(key)) {
let insertIndex = lowerBound(cache.timestamps, timestamp);
if (cache.timestamps[insertIndex] !== timestamp) {
if (insertIndex === cache.timestamps.length) {
cache.timestamps.push(timestamp);
} else {
cache.timestamps.splice(insertIndex, 0, timestamp);
}
}
cache.timestamps.splice(insertIndex, 0, timestamp);
}
});
}
Expand Down
97 changes: 43 additions & 54 deletions src/shared/log/LogField.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// license that can be found in the LICENSE file
// at the root directory of this project.

import { lowerBound, upperBound } from "../util";
import LoggableType from "./LoggableType";
import { isChildLog } from "./LogKeyUtils";
import { logValuesEqual } from "./LogUtil";
Expand Down Expand Up @@ -86,40 +87,34 @@ export default class LogField {
cacheIndex = this.getRangeCache[uuid];
}

let startValueIndex = -1;
if (
let startSearchIndex =
cacheIndex !== null &&
cacheIndex >= 0 &&
cacheIndex < this.data.timestamps.length &&
!(this.data.timestamps[cacheIndex] > start)
) {
// Search from previous location
let rawIndex = this.data.timestamps.slice(cacheIndex).findIndex((x) => x > start);
if (rawIndex !== -1) startValueIndex = rawIndex + cacheIndex;
}
if (startValueIndex === -1) {
// Search from start
startValueIndex = this.data.timestamps.findIndex((x) => x > start);
}
if (startValueIndex === -1) {
startValueIndex = this.data.timestamps.length - 1;
} else if (startValueIndex !== 0) {
startValueIndex -= 1;
}
? cacheIndex + 1
: 0;
let startUpper = upperBound(this.data.timestamps, start, startSearchIndex);
let startValueIndex = startUpper > 0 ? startUpper - 1 : 0;
if (startOffset !== undefined && startOffset >= -startValueIndex) {
startValueIndex += startOffset;
}

let endValueIndex = -1;
if (cacheIndex !== null && cacheIndex < this.data.timestamps.length && !(this.data.timestamps[cacheIndex] >= end)) {
// Search from previous location
let rawIndex = this.data.timestamps.slice(cacheIndex).findIndex((x) => x >= end);
if (rawIndex !== -1) endValueIndex = rawIndex + cacheIndex;
}
if (endValueIndex === -1) {
// Search from start
endValueIndex = this.data.timestamps.findIndex((x) => x >= end);
let endValueIndex: number;
if (start === end) {
if (startUpper === 0 || this.data.timestamps[startUpper - 1] < end) {
endValueIndex = startUpper;
} else if (startUpper === 1 || this.data.timestamps[startUpper - 2] < end) {
endValueIndex = startUpper - 1;
} else {
endValueIndex = lowerBound(this.data.timestamps, end);
}
} else if (end > start) {
endValueIndex = lowerBound(this.data.timestamps, end, startUpper);
} else {
endValueIndex = lowerBound(this.data.timestamps, end);
}
if (endValueIndex === -1 || endValueIndex === this.data.timestamps.length - 1) {
if (endValueIndex >= this.data.timestamps.length - 1) {
// Extend to end of timestamps
timestamps = this.data.timestamps.slice(startValueIndex);
values = this.data.values.slice(startValueIndex);
Expand Down Expand Up @@ -180,40 +175,34 @@ export default class LogField {
if (value === null) return;

// Find position to insert based on timestamp
let insertIndex: number;
if (this.data.timestamps.length > 0 && timestamp > this.data.timestamps[this.data.timestamps.length - 1]) {
// There's a good chance this data is at the end of the log, so check that first
insertIndex = this.data.timestamps.length;
} else {
// Adding in the middle, find where to insert it
let alreadyExists = false;
insertIndex =
this.data.timestamps.findLastIndex((x) => {
if (alreadyExists) return;
if (x === timestamp) alreadyExists = true;
return x < timestamp;
}) + 1;
if (alreadyExists) {
this.data.values[this.data.timestamps.indexOf(timestamp)] = value;
return;
}
let timestamps = this.data.timestamps;
let values = this.data.values;
let length = timestamps.length;
if (length > 0 && timestamps[length - 1] === timestamp) {
values[length - 1] = value;
return;
}
let insertIndex = lowerBound(timestamps, timestamp);
if (insertIndex < length && timestamps[insertIndex] === timestamp) {
values[insertIndex] = value;
return;
}

// Compare to adjacent values
let childLog = isChildLog(this.wpilibType, this.structuredType);
if (!childLog && insertIndex > 0 && logValuesEqual(this.type, value, this.data.values[insertIndex - 1])) {
let childLog = this.type === LoggableType.Raw && isChildLog(this.wpilibType, this.structuredType);
if (!childLog && insertIndex > 0 && logValuesEqual(this.type, value, values[insertIndex - 1])) {
// Same as the previous value
} else if (
!childLog &&
insertIndex < this.data.values.length &&
logValuesEqual(this.type, value, this.data.values[insertIndex])
) {
} else if (!childLog && insertIndex < values.length && logValuesEqual(this.type, value, values[insertIndex])) {
// Same as the next value
this.data.timestamps[insertIndex] = timestamp;
timestamps[insertIndex] = timestamp;
} else if (insertIndex === length) {
// New value at the end
timestamps.push(timestamp);
values.push(value);
} else {
// New value
this.data.timestamps.splice(insertIndex, 0, timestamp);
this.data.values.splice(insertIndex, 0, value);
// New value in the middle
timestamps.splice(insertIndex, 0, timestamp);
values.splice(insertIndex, 0, value);
}
}

Expand Down
11 changes: 9 additions & 2 deletions src/shared/log/LogUtil.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,15 @@ export function logValuesEqual(type: LoggableType, a: any, b: any): boolean {
case LoggableType.NumberArray:
case LoggableType.StringArray:
return arraysEqual(a, b);
case LoggableType.Raw:
return arraysEqual(Array.from(a as Uint8Array), Array.from(b as Uint8Array));
case LoggableType.Raw: {
let aRaw = a as Uint8Array;
let bRaw = b as Uint8Array;
if (aRaw.byteLength !== bRaw.byteLength) return false;
for (let i = 0; i < aRaw.byteLength; i++) {
if (aRaw[i] !== bRaw[i]) return false;
}
return true;
}
default:
return false;
}
Expand Down
5 changes: 2 additions & 3 deletions src/shared/renderers/ConsoleRenderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import { AnsiUp } from "ansi_up";
import { SelectionMode } from "../Selection";
import { IS_LITE } from "../buildConstants";
import { formatTimeWithMS } from "../util";
import { formatTimeWithMS, upperBound } from "../util";
import TabRenderer from "./TabRenderer";

export default class ConsoleRenderer implements TabRenderer {
Expand Down Expand Up @@ -369,8 +369,7 @@ export default class ConsoleRenderer implements TabRenderer {
let highlight = (time: number | null, className: string) => {
Array.from(this.TABLE_BODY.children).forEach((row) => row.classList.remove(className));
if (time) {
let target = this.renderedTimestamps.findIndex((value) => value > time);
if (target === -1) target = this.renderedTimestamps.length; // Use the last timestamp
let target = upperBound(this.renderedTimestamps, time);
if (target < 1) target = 1; // Use the first timestamp (0 is the header cell, so the indexes of console cells start at 1)
this.TABLE_BODY.children[target].classList.add(className);
}
Expand Down
13 changes: 7 additions & 6 deletions src/shared/renderers/TableRenderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { SelectionMode } from "../Selection";
import { getLogValueText } from "../log/LogUtil";
import { LogValueSetAny } from "../log/LogValueSets";
import LoggableType from "../log/LoggableType";
import { arraysEqual, createUUID, formatTimeWithMS } from "../util";
import { arraysEqual, createUUID, formatTimeWithMS, lowerBound, upperBound } from "../util";
import TabRenderer from "./TabRenderer";

export default class TableRenderer implements TabRenderer {
Expand Down Expand Up @@ -107,7 +107,7 @@ export default class TableRenderer implements TabRenderer {
*/
private scrollToSelected() {
if (this.selectedTime !== null) {
let targetRow = this.timestamps.findLastIndex((timestamp) => timestamp <= this.selectedTime!);
let targetRow = upperBound(this.timestamps, this.selectedTime!) - 1;
if (targetRow === -1) targetRow = 0;

const visibleHeight = this.TABLE_CONTAINER.clientHeight - this.TABLE_BODY.firstElementChild!.clientHeight;
Expand Down Expand Up @@ -170,7 +170,7 @@ export default class TableRenderer implements TabRenderer {
let highlight = (time: number | null, className: string) => {
this.dataRows.forEach((row) => row.classList.remove(className));
if (time) {
let dataRowIndex = this.dataRowTimestamps.findLastIndex((timestamp) => timestamp <= time);
let dataRowIndex = upperBound(this.dataRowTimestamps, time) - 1;
if (dataRowIndex !== -1 && dataRowIndex < this.dataRows.length) {
this.dataRows[dataRowIndex].classList.add(className);
}
Expand Down Expand Up @@ -258,7 +258,8 @@ export default class TableRenderer implements TabRenderer {
}

// Update timestamps, count removed rows from start
let removedRows = this.timestamps.findIndex((timestamp) => timestamp === command.timestamps[1]) - 1;
let matchIndex = command.timestamps.length > 1 ? lowerBound(this.timestamps, command.timestamps[1]) : -1;
let removedRows = matchIndex !== -1 && this.timestamps[matchIndex] === command.timestamps[1] ? matchIndex - 1 : -2;
if (removedRows > 0) {
this.TABLE_CONTAINER.scrollTop = initialScrollPosition - removedRows * this.ROW_HEIGHT_PX;
}
Expand Down Expand Up @@ -320,9 +321,9 @@ export default class TableRenderer implements TabRenderer {
cellText[i - dataRowStart].push("null");
}
} else {
let nextIndex = 0;
for (let i = dataRowStart; i < dataRowEnd; i++) {
let nextIndex = field.data!.timestamps.findIndex((value) => value > this.timestamps[i]);
if (nextIndex === -1) nextIndex = field.data!.timestamps.length;
nextIndex = upperBound(field.data!.timestamps, this.timestamps[i], nextIndex);
if (nextIndex === 0 || field.type === null) {
cellText[i - dataRowStart].push("null");
} else {
Expand Down
Loading
Loading