Skip to content

Advanced message parsing #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+70 −33
Open

Advanced message parsing #34

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cab9299
feat(Parser): allow left-padded values
leon0399 Feb 27, 2024
aff3e74
feat(Parser): allow equals (`=`) sign for label-value separator
leon0399 Feb 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 43 additions & 18 deletions src/msgAggregatorWorker.test.ts
View file
Original file line number Diff line number Diff line change
@@ -81,24 +81,49 @@ describe("Parsing data", () => {
);
});

test("labeled", () => {
const messages = [
`0${trailingFieldDelimiter}${recordDelimiter}`,
`label_1:1${fieldDelimiter}label_2:2${trailingFieldDelimiter}${recordDelimiter}`,
`label_1:3${fieldDelimiter}label_2:4${trailingFieldDelimiter}${recordDelimiter}`,
];

const assertion = {
datasetNames: ["label_1", "label_2"],
parsedLines: [
{ label_1: 1, label_2: 2 },
{ label_1: 3, label_2: 4 },
],
};

expect(messageAggregator.parseSerialMessages(messages)).toEqual(
assertion
);
describe.each([
["colon", ":"],
["equals", "="],
])("%s label delimiter", (_, labelDelimiter) => {
test("labeled", () => {
const messages = [
`0${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter}1${fieldDelimiter}label_2${labelDelimiter}2${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter}3${fieldDelimiter}label_2${labelDelimiter}4${trailingFieldDelimiter}${recordDelimiter}`,
];

const assertion = {
datasetNames: ["label_1", "label_2"],
parsedLines: [
{ label_1: 1, label_2: 2 },
{ label_1: 3, label_2: 4 },
],
};

expect(messageAggregator.parseSerialMessages(messages)).toEqual(
assertion
);
});

test("labeled padded", () => {
const messages = [
`0${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter} 1${fieldDelimiter}label_2${labelDelimiter} 20${trailingFieldDelimiter}${recordDelimiter}`,
`label_1${labelDelimiter} 300${fieldDelimiter}label_2${labelDelimiter}4000${trailingFieldDelimiter}${recordDelimiter}`,
];

const assertion = {
datasetNames: ["label_1", "label_2"],
parsedLines: [
{ label_1: 1, label_2: 20 },
{ label_1: 300, label_2: 4000 },
],
};

expect(messageAggregator.parseSerialMessages(messages)).toEqual(
assertion
);
});
});

test("buffering", () => {
42 changes: 27 additions & 15 deletions src/msgAggregatorWorker.ts
View file
Original file line number Diff line number Diff line change
@@ -18,9 +18,9 @@ ctx.addEventListener("message", (event) => {

let buffer = "";
let discardFirstLine = true;
const separator = "\r?\n";
const lineSeparator = "\r?\n";
const delimiter = "[, \t]+"; // Serial Plotter protocol supports Comma, Space & Tab characters as delimiters
var separatorRegex = new RegExp(`(${separator})`, "g");
var lineSeparatorRegex = new RegExp(`(${lineSeparator})`, "g");
var delimiterRegex = new RegExp(delimiter, "g");

export const parseSerialMessages = (
@@ -33,8 +33,8 @@ export const parseSerialMessages = (
// so we need to discard it and start aggregating from the first encountered separator
let joinMessages = messages.join("");
if (discardFirstLine) {
separatorRegex.lastIndex = 0; // Reset lastIndex to ensure match happens from beginning of string
const separatorMatch = separatorRegex.exec(joinMessages);
lineSeparatorRegex.lastIndex = 0; // Reset lastIndex to ensure match happens from beginning of string
const separatorMatch = lineSeparatorRegex.exec(joinMessages);
if (separatorMatch && separatorMatch.index > -1) {
joinMessages = joinMessages.substring(
separatorMatch.index + separatorMatch[0].length
@@ -50,14 +50,16 @@ export const parseSerialMessages = (

//add any leftover from the buffer to the first line
const messagesAndBuffer = ((buffer || "") + joinMessages)
.split(separatorRegex)
.split(lineSeparatorRegex)
.filter((message) => message.length > 0);

// remove the previous buffer
buffer = "";
separatorRegex.lastIndex = 0;
lineSeparatorRegex.lastIndex = 0;
// check if the last message contains the delimiter, if not, it's an incomplete string that needs to be added to the buffer
if (!separatorRegex.test(messagesAndBuffer[messagesAndBuffer.length - 1])) {
if (
!lineSeparatorRegex.test(messagesAndBuffer[messagesAndBuffer.length - 1])
) {
buffer = messagesAndBuffer[messagesAndBuffer.length - 1];
messagesAndBuffer.splice(-1);
}
@@ -66,10 +68,17 @@ export const parseSerialMessages = (
const parsedLines: { [key: string]: number }[] = [];

// for each line, explode variables
separatorRegex.lastIndex = 0;
lineSeparatorRegex.lastIndex = 0;
messagesAndBuffer
.filter((message) => !separatorRegex.test(message))
.filter((message) => !lineSeparatorRegex.test(message))
.forEach((message) => {
// replace all delimiters with a single space for uniform parsing
message = message.replace(delimiterRegex, " ");
// replace multiple spaces with a single space
message = message.replace(/\s+/g, " ");
// replace all equal signs with a colon
message = message.replace(/=/g, ":");

const parsedLine: { [key: string]: number } = {};

// Part Separator symbols i.e. Space, Tab & Comma are fully supported
@@ -80,12 +89,15 @@ export const parseSerialMessages = (
// if we find a colon, we assume the latter is being used
let tokens: string[] = [];
if (message.indexOf(":") > 0) {
message.split(delimiterRegex).forEach((keyValue: string) => {
let [key, value] = keyValue.split(":");
key = key && key.trim();
value = value && value.trim();
if (key && key.length > 0 && value && value.length > 0) {
tokens.push(...[key, value]);
// Splitting by the separator and handling possible spaces
const keyValuePairs = message.split(":").map((kv) => kv.trim());
let reformedLine = keyValuePairs.join(":").split(delimiterRegex);

reformedLine.forEach((kv) => {
const [key, value] = kv.split(":");
if (key && value) {
tokens.push(key.trim());
tokens.push(value.trim());
}
});
} else {