feat(decoder): Add support for Structured CLP IR streams but without log-level filtering.

src/services/LogFileManager/index.ts

@@ -1,7 +1,7 @@
 /* eslint max-lines: ["error", 400] */
 import {
     Decoder,
-    DecoderOptionsType,
+    DecoderOptions,
 } from "../../typings/decoders";
 import {MAX_V8_STRING_LENGTH} from "../../typings/js";
 import {LogLevelFilter} from "../../typings/logs";
@@ -110,7 +110,7 @@ class LogFileManager {
     static async create (
         fileSrc: FileSrcType,
         pageSize: number,
-        decoderOptions: DecoderOptionsType,
+        decoderOptions: DecoderOptions,
         onQueryResults: (queryResults: QueryResults) => void,
     ): Promise<LogFileManager> {
         const {fileName, fileData} = await loadFile(fileSrc);
@@ -138,13 +138,13 @@ class LogFileManager {
     static async #initDecoder (
         fileName: string,
         fileData: Uint8Array,
-        decoderOptions: DecoderOptionsType
+        decoderOptions: DecoderOptions
     ): Promise<Decoder> {
         let decoder: Decoder;
         if (fileName.endsWith(".jsonl")) {
             decoder = new JsonlDecoder(fileData, decoderOptions);
         } else if (fileName.endsWith(".clp.zst")) {
-            decoder = await ClpIrDecoder.create(fileData);
+            decoder = await ClpIrDecoder.create(fileData, decoderOptions);
         } else {
             throw new Error(`No decoder supports ${fileName}`);
         }
@@ -161,7 +161,7 @@ class LogFileManager {
     /* Sets any formatter options that exist in the decoder's options.
      * @param options
      */
-    setFormatterOptions (options: DecoderOptionsType) {
+    setFormatterOptions (options: DecoderOptions) {
         this.#decoder.setFormatterOptions(options);
     }
 

src/services/MainWorker.ts

@@ -1,4 +1,5 @@
 import dayjs from "dayjs";
+import dayjsBigIntSupport from "dayjs/plugin/bigIntSupport";
 import dayjsTimezone from "dayjs/plugin/timezone";
 import dayjsUtc from "dayjs/plugin/utc";
 
@@ -17,6 +18,7 @@ import LogFileManager from "./LogFileManager";
 /* eslint-disable import/no-named-as-default-member */
 dayjs.extend(dayjsUtc);
 dayjs.extend(dayjsTimezone);
+dayjs.extend(dayjsBigIntSupport);
 /* eslint-enable import/no-named-as-default-member */
 
 /**

src/services/decoders/ClpIrDecoder.ts

@@ -1,32 +1,61 @@
-import clpFfiJsModuleInit, {ClpIrStreamReader} from "clp-ffi-js";
-
+import clpFfiJsModuleInit, {ClpStreamReader} from "../../../deps/ClpFfiJs-worker";
 import {Nullable} from "../../typings/common";
 import {
+    ClpIrDecoderOptions,
     Decoder,
     DecodeResultType,
     FilteredLogEventMap,
     LogEventCount,
 } from "../../typings/decoders";
-import {LogLevelFilter} from "../../typings/logs";
+import {JsonObject} from "../../typings/js";
+import {
+    LOG_LEVEL,
+    LogLevelFilter,
+} from "../../typings/logs";
+import LogbackFormatter from "../formatters/LogbackFormatter";
+import {
+    convertToDayjsTimestamp,
+    isJsonObject,
+} from "./JsonlDecoder/utils";
 
 
 class ClpIrDecoder implements Decoder {
-    #streamReader: ClpIrStreamReader;
+    #streamReader: ClpStreamReader;
+
+    #formatter: Nullable<LogbackFormatter>;
+
+    #logLevelKey: Nullable<string>;
 
-    constructor (streamReader: ClpIrStreamReader) {
+    constructor (streamReader: ClpStreamReader, formatter: Nullable<LogbackFormatter>, logLevelKey: Nullable<string>) {
         this.#streamReader = streamReader;
+        this.#formatter = formatter;
+        this.#logLevelKey = logLevelKey;
     }
 
     /**
      * Creates a new ClpIrDecoder instance.
      *
      * @param dataArray The input data array to be passed to the decoder.
+     * @param decoderOptions
      * @return The created ClpIrDecoder instance.
      */
-    static async create (dataArray: Uint8Array): Promise<ClpIrDecoder> {
+    static async create (
+        dataArray: Uint8Array,
+        decoderOptions: ClpIrDecoderOptions
+    ): Promise<ClpIrDecoder> {
         const module = await clpFfiJsModuleInit();
-        const streamReader = new module.ClpIrStreamReader(dataArray);
-        return new ClpIrDecoder(streamReader);
+        const streamReader = new module.ClpStreamReader(dataArray, decoderOptions);
+
+        let formatter: Nullable<LogbackFormatter> = null;
+        let logLevelKey: Nullable<string> = null;
+        if (
+            module.IRProtocolErrorCode.SUPPORTED === streamReader.getIrProtocolErrorCode()
+        ) {
+            formatter = new LogbackFormatter({formatString: decoderOptions.formatString});
+            ({logLevelKey} = decoderOptions);
+        }
+
+        return new ClpIrDecoder(streamReader, formatter, logLevelKey);
     }
 
     getEstimatedNumEvents (): number {
@@ -60,7 +89,36 @@
         endIdx: number,
         useFilter: boolean
     ): Nullable<DecodeResultType[]> {
-        return this.#streamReader.decodeRange(beginIdx, endIdx, useFilter);
+        const results = this.#streamReader.decodeRange(beginIdx, endIdx, useFilter);
+        if (null === this.#formatter) {
+            return results;
+        }
+        results.forEach((result) => {
+            const [message, timestamp] = result;
+            let fields: JsonObject = {};
+            try {
+                fields = JSON.parse(message) as JsonObject;
+                if (false === isJsonObject(fields)) {
+                    throw new Error("Unexpected non-object.");
+                } else if (null !== this.#logLevelKey) {
+                    const logLevel = fields[this.#logLevelKey];
+                    fields[this.#logLevelKey] = LOG_LEVEL[logLevel as LOG_LEVEL];
+                }
+            } catch (e) {
+                if (0 === message.length) {
+                    return;
+                }
+                console.error(e, message);
+            }
+
+            // @ts-expect-error `this.#formatter` is certainly non-null
+            result[0] = this.#formatter.formatLogEvent({
+                fields: fields,
+                timestamp: convertToDayjsTimestamp(timestamp),
+            });
+        });
+
+        return results;
     }
 }
 

src/services/decoders/JsonlDecoder/index.ts

@@ -5,7 +5,7 @@ import {
     Decoder,
     DecodeResultType,
     FilteredLogEventMap,
-    JsonlDecoderOptionsType,
+    JsonlDecoderOptions,
     LogEventCount,
 } from "../../../typings/decoders";
 import {Formatter} from "../../../typings/formatters";
@@ -25,7 +25,7 @@ import {
 
 
 /**
- * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptionsType` for
+ * A decoder for JSONL (JSON lines) files that contain log events. See `JsonlDecoderOptions` for
  * properties that are specific to log events (compared to generic JSON records).
  */
 class JsonlDecoder implements Decoder {
@@ -49,7 +49,7 @@ class JsonlDecoder implements Decoder {
      * @param dataArray
      * @param decoderOptions
      */
-    constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptionsType) {
+    constructor (dataArray: Uint8Array, decoderOptions: JsonlDecoderOptions) {
         this.#dataArray = dataArray;
         this.#logLevelKey = decoderOptions.logLevelKey;
         this.#timestampKey = decoderOptions.timestampKey;
@@ -81,7 +81,7 @@ class JsonlDecoder implements Decoder {
         };
     }
 
-    setFormatterOptions (options: JsonlDecoderOptionsType): boolean {
+    setFormatterOptions (options: JsonlDecoderOptions): boolean {
         this.#formatter = new LogbackFormatter({formatString: options.formatString});
 
         return true;

src/services/decoders/JsonlDecoder/utils.ts

@@ -53,12 +53,13 @@ const convertToLogLevelValue = (field: JsonValue | undefined): LOG_LEVEL => {
  * - the timestamp's value is an unsupported type.
  * - the timestamp's value is not a valid dayjs timestamp.
  */
-const convertToDayjsTimestamp = (field: JsonValue | undefined): dayjs.Dayjs => {
+const convertToDayjsTimestamp = (field: JsonValue | bigint | undefined): dayjs.Dayjs => {
     // If the field is an invalid type, then set the timestamp to `INVALID_TIMESTAMP_VALUE`.
     // NOTE: dayjs surprisingly thinks `undefined` is a valid date. See
     // https://day.js.org/docs/en/parse/now#docsNav
     if (("string" !== typeof field &&
-        "number" !== typeof field) ||
+        "number" !== typeof field &&
+        "bigint" !== typeof field) ||
         "undefined" === typeof field
     ) {
         // `INVALID_TIMESTAMP_VALUE` is a valid dayjs date. Another potential option is

src/typings/config.ts

@@ -1,4 +1,4 @@
-import {JsonlDecoderOptionsType} from "./decoders";
+import {JsonlDecoderOptions} from "./decoders";
 import {TAB_NAME} from "./tab";
 
 
@@ -27,7 +27,7 @@ enum LOCAL_STORAGE_KEY {
 /* eslint-enable @typescript-eslint/prefer-literal-enum-member */
 
 type ConfigMap = {
-    [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptionsType,
+    [CONFIG_KEY.DECODER_OPTIONS]: JsonlDecoderOptions,
     [CONFIG_KEY.INITIAL_TAB_NAME]: TAB_NAME,
     [CONFIG_KEY.THEME]: THEME_NAME,
     [CONFIG_KEY.PAGE_SIZE]: number,

src/typings/decoders.ts

@@ -14,13 +14,25 @@ interface LogEventCount {
  * @property logLevelKey The key of the kv-pair that contains the log level in every record.
  * @property timestampKey The key of the kv-pair that contains the timestamp in every record.
  */
-interface JsonlDecoderOptionsType {
+interface JsonlDecoderOptions {
     formatString: string,
     logLevelKey: string,
     timestampKey: string,
 }
 
-type DecoderOptionsType = JsonlDecoderOptionsType;
+/**
+ * Options for the Structured CLP IR decoder.
+ */
+interface StructuredClpIrDecoderOptions extends JsonlDecoderOptions {
+}
+
+/**
+ * Options for the CLP IR decoder. Currently, those options are only effective if the stream is
+ * Structured IR.
+ */
+type ClpIrDecoderOptions = StructuredClpIrDecoderOptions;
+
+type DecoderOptions = JsonlDecoderOptions | ClpIrDecoderOptions;
 
 /**
  * Type of the decoded log event. We use an array rather than object so that it's easier to return
@@ -85,7 +97,7 @@ interface Decoder {
      * @param options
      * @return Whether the options were successfully set.
      */
-    setFormatterOptions(options: DecoderOptionsType): boolean;
+    setFormatterOptions(options: DecoderOptions): boolean;
 
     /**
      * Decodes log events in the range `[beginIdx, endIdx)` of the filtered or unfiltered
@@ -106,10 +118,11 @@ interface Decoder {
 
 export type {
     ActiveLogCollectionEventIdx,
+    ClpIrDecoderOptions,
     Decoder,
     DecodeResultType,
-    DecoderOptionsType,
+    DecoderOptions,
     FilteredLogEventMap,
-    JsonlDecoderOptionsType,
+    JsonlDecoderOptions,
     LogEventCount,
 };

src/typings/worker.ts

@@ -1,7 +1,7 @@
 import {Nullable} from "./common";
 import {
     ActiveLogCollectionEventIdx,
-    DecoderOptionsType,
+    DecoderOptions,
 } from "./decoders";
 import {
     LOG_LEVEL,
@@ -89,7 +89,7 @@ type WorkerReqMap = {
         fileSrc: FileSrcType,
         pageSize: number,
         cursor: CursorType,
-        decoderOptions: DecoderOptionsType
+        decoderOptions: DecoderOptions
     },
     [WORKER_REQ_CODE.LOAD_PAGE]: {
         cursor: CursorType,

src/utils/config.ts

@@ -6,7 +6,7 @@ import {
     LOCAL_STORAGE_KEY,
     THEME_NAME,
 } from "../typings/config";
-import {DecoderOptionsType} from "../typings/decoders";
+import {DecoderOptions} from "../typings/decoders";
 import {TAB_NAME} from "../typings/tab";
 
 
@@ -131,7 +131,7 @@ const getConfig = <T extends CONFIG_KEY>(key: T): ConfigMap[T] => {
                 timestampKey: window.localStorage.getItem(
                     LOCAL_STORAGE_KEY.DECODER_OPTIONS_TIMESTAMP_KEY
                 ),
-            } as DecoderOptionsType;
+            } as DecoderOptions;
             break;
         case CONFIG_KEY.INITIAL_TAB_NAME:
             value = window.localStorage.getItem(LOCAL_STORAGE_KEY.INITIAL_TAB_NAME);