repl: extract and standardize history from both repl and interface

Author: puskin

doc/api/repl.md

@@ -649,10 +649,31 @@ called from within the action function for commands registered using the
 
 <!-- YAML
 added: v11.10.0
+changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/58225
+    description: Updated the `historyPath` parameter to accept an object
+                 with `historyFile`, `historySize`, `removeHistoryDuplicates` and
+                 `onHistoryFileLoaded` properties.
 -->
 
-* `historyPath` {string} the path to the history file
+* `historyPath` {Object|string} the path to the history file
+  If it is a string, it is the path to the history file.
+  If it is an object, it can have the following properties:
+  * `historyFile` {string} the path to the history file
+  * `historySize` {number} Maximum number of history lines retained. To disable
+    the history set this value to `0`. This option makes sense only if
+    `terminal` is set to `true` by the user or by an internal `output` check,
+    otherwise the history caching mechanism is not initialized at all.
+    **Default:** `30`.
+  * `removeHistoryDuplicates` {boolean} If `true`, when a new input line added
+    to the history list duplicates an older one, this removes the older line
+    from the list. **Default:** `false`.
+  * `onHistoryFileLoaded` {Function} called when history writes are ready or upon error
+    * `err` {Error}
+    * `repl` {repl.REPLServer}
 * `callback` {Function} called when history writes are ready or upon error
+  (Optional if provided as `onHistoryFileLoaded` in `historyPath`)
   * `err` {Error}
   * `repl` {repl.REPLServer}
 

lib/internal/readline/interface.js

@@ -3,14 +3,12 @@
 const {
   ArrayFrom,
   ArrayPrototypeFilter,
-  ArrayPrototypeIndexOf,
   ArrayPrototypeJoin,
   ArrayPrototypeMap,
   ArrayPrototypePop,
   ArrayPrototypePush,
   ArrayPrototypeReverse,
   ArrayPrototypeShift,
-  ArrayPrototypeSplice,
   ArrayPrototypeUnshift,
   DateNow,
   FunctionPrototypeCall,
@@ -19,6 +17,7 @@ const {
   MathMax,
   MathMaxApply,
   NumberIsFinite,
+  ObjectDefineProperty,
   ObjectSetPrototypeOf,
   RegExpPrototypeExec,
   SafeStringIterator,
@@ -30,7 +29,6 @@ const {
   StringPrototypeSlice,
   StringPrototypeSplit,
   StringPrototypeStartsWith,
-  StringPrototypeTrim,
   Symbol,
   SymbolAsyncIterator,
   SymbolDispose,
@@ -46,8 +44,6 @@ const {
 
 const {
   validateAbortSignal,
-  validateArray,
-  validateNumber,
   validateString,
   validateUint32,
 } = require('internal/validators');
@@ -64,7 +60,6 @@ const {
   charLengthLeft,
   commonPrefix,
   kSubstringSearch,
-  reverseString,
 } = require('internal/readline/utils');
 let emitKeypressEvents;
 let kFirstEventParam;
@@ -75,8 +70,8 @@ const {
 } = require('internal/readline/callbacks');
 
 const { StringDecoder } = require('string_decoder');
+const { ReplHistory } = require('internal/repl/history');
 
-const kHistorySize = 30;
 const kMaxUndoRedoStackSize = 2048;
 const kMincrlfDelay = 100;
 /**
@@ -150,7 +145,6 @@ const kWriteToOutput = Symbol('_writeToOutput');
 const kYank = Symbol('_yank');
 const kYanking = Symbol('_yanking');
 const kYankPop = Symbol('_yankPop');
-const kNormalizeHistoryLineEndings = Symbol('_normalizeHistoryLineEndings');
 const kSavePreviousState = Symbol('_savePreviousState');
 const kRestorePreviousState = Symbol('_restorePreviousState');
 const kPreviousLine = Symbol('_previousLine');
@@ -172,9 +166,6 @@ function InterfaceConstructor(input, output, completer, terminal) {
 
   FunctionPrototypeCall(EventEmitter, this);
 
-  let history;
-  let historySize;
-  let removeHistoryDuplicates = false;
   let crlfDelay;
   let prompt = '> ';
   let signal;
@@ -184,14 +175,17 @@ function InterfaceConstructor(input, output, completer, terminal) {
     output = input.output;
     completer = input.completer;
     terminal = input.terminal;
-    history = input.history;
-    historySize = input.historySize;
     signal = input.signal;
+
+    // It is possible to configure the history through the input object
+    const historySize = input.historySize;
+    const history = input.history;
+    const removeHistoryDuplicates = input.removeHistoryDuplicates;
+
     if (input.tabSize !== undefined) {
       validateUint32(input.tabSize, 'tabSize', true);
       this.tabSize = input.tabSize;
     }
-    removeHistoryDuplicates = input.removeHistoryDuplicates;
     if (input.prompt !== undefined) {
       prompt = input.prompt;
     }
@@ -212,24 +206,18 @@ function InterfaceConstructor(input, output, completer, terminal) {
 
     crlfDelay = input.crlfDelay;
     input = input.input;
-  }
 
-  if (completer !== undefined && typeof completer !== 'function') {
-    throw new ERR_INVALID_ARG_VALUE('completer', completer);
+    input.historySize = historySize;
+    input.history = history;
+    input.removeHistoryDuplicates = removeHistoryDuplicates;
   }
 
-  if (history === undefined) {
-    history = [];
-  } else {
-    validateArray(history, 'history');
-  }
+  this.setupHistoryManager(input);
 
-  if (historySize === undefined) {
-    historySize = kHistorySize;
+  if (completer !== undefined && typeof completer !== 'function') {
+    throw new ERR_INVALID_ARG_VALUE('completer', completer);
   }
 
-  validateNumber(historySize, 'historySize', 0);
-
   // Backwards compat; check the isTTY prop of the output stream
   //  when `terminal` was not specified
   if (terminal === undefined && !(output === null || output === undefined)) {
@@ -245,8 +233,6 @@ function InterfaceConstructor(input, output, completer, terminal) {
   this.input = input;
   this[kUndoStack] = [];
   this[kRedoStack] = [];
-  this.history = history;
-  this.historySize = historySize;
   this[kPreviousCursorCols] = -1;
 
   // The kill ring is a global list of blocks of text that were previously
@@ -257,7 +243,6 @@ function InterfaceConstructor(input, output, completer, terminal) {
   this[kKillRing] = [];
   this[kKillRingCursor] = 0;
 
-  this.removeHistoryDuplicates = !!removeHistoryDuplicates;
   this.crlfDelay = crlfDelay ?
     MathMax(kMincrlfDelay, crlfDelay) :
     kMincrlfDelay;
@@ -267,7 +252,6 @@ function InterfaceConstructor(input, output, completer, terminal) {
 
   this.terminal = !!terminal;
 
-
   function onerror(err) {
     self.emit('error', err);
   }
@@ -346,8 +330,6 @@ function InterfaceConstructor(input, output, completer, terminal) {
     // Cursor position on the line.
     this.cursor = 0;
 
-    this.historyIndex = -1;
-
     if (output !== null && output !== undefined)
       output.on('resize', onresize);
 
@@ -400,6 +382,36 @@ class Interface extends InterfaceConstructor {
     return this[kPrompt];
   }
 
+  setupHistoryManager(options) {
+    this.historyManager = new ReplHistory(this, options);
+
+    if (options.onHistoryFileLoaded) {
+      this.historyManager.initialize(options.onHistoryFileLoaded);
+    }
+
+    ObjectDefineProperty(this, 'history', {
+      __proto__: null, configurable: true, enumerable: true,
+      get() { return this.historyManager.getHistory(); },
+      set(newHistory) { return this.historyManager.setHistory(newHistory); },
+    });
+
+    ObjectDefineProperty(this, 'historyIndex', {
+      __proto__: null, configurable: true, enumerable: true,
+      get() { return this.historyManager.getHistoryIndex(); },
+      set(historyIndex) { return this.historyManager.setHistoryIndex(historyIndex); },
+    });
+
+    ObjectDefineProperty(this, 'historySize', {
+      __proto__: null, configurable: true, enumerable: true,
+      get() { return this.historyManager.getSize(); },
+    });
+
+    ObjectDefineProperty(this, '_flushing', {
+      __proto__: null, configurable: true, enumerable: true,
+      get() { return this.historyManager.isFlushing(); },
+    });
+  }
+
   [kSetRawMode](mode) {
     const wasInRawMode = this.input.isRaw;
 
@@ -475,70 +487,8 @@ class Interface extends InterfaceConstructor {
     }
   }
 
-  // Convert newlines to a consistent format for history storage
-  [kNormalizeHistoryLineEndings](line, from, to, reverse = true) {
-    // Multiline history entries are saved reversed
-    // History is structured with the newest entries at the top
-    // and the oldest at the bottom. Multiline histories, however, only occupy
-    // one line in the history file. When loading multiline history with
-    // an old node binary, the history will be saved in the old format.
-    // This is why we need to reverse the multilines.
-    // Reversing the multilines is necessary when adding / editing and displaying them
-    if (reverse) {
-      // First reverse the lines for proper order, then convert separators
-      return reverseString(line, from, to);
-    }
-    // For normal cases (saving to history or non-multiline entries)
-    return StringPrototypeReplaceAll(line, from, to);
-  }
-
   [kAddHistory]() {
-    if (this.line.length === 0) return '';
-
-    // If the history is disabled then return the line
-    if (this.historySize === 0) return this.line;
-
-    // If the trimmed line is empty then return the line
-    if (StringPrototypeTrim(this.line).length === 0) return this.line;
-
-    // This is necessary because each line would be saved in the history while creating
-    // A new multiline, and we don't want that.
-    if (this[kIsMultiline] && this.historyIndex === -1) {
-      ArrayPrototypeShift(this.history);
-    } else if (this[kLastCommandErrored]) {
-      // If the last command errored and we are trying to edit the history to fix it
-      // Remove the broken one from the history
-      ArrayPrototypeShift(this.history);
-    }
-
-    const normalizedLine = this[kNormalizeHistoryLineEndings](this.line, '\n', '\r', true);
-
-    if (this.history.length === 0 || this.history[0] !== normalizedLine) {
-      if (this.removeHistoryDuplicates) {
-        // Remove older history line if identical to new one
-        const dupIndex = ArrayPrototypeIndexOf(this.history, this.line);
-        if (dupIndex !== -1) ArrayPrototypeSplice(this.history, dupIndex, 1);
-      }
-
-      // Add the new line to the history
-      ArrayPrototypeUnshift(this.history, normalizedLine);
-
-      // Only store so many
-      if (this.history.length > this.historySize)
-        ArrayPrototypePop(this.history);
-    }
-
-    this.historyIndex = -1;
-
-    // The listener could change the history object, possibly
-    // to remove the last added entry if it is sensitive and should
-    // not be persisted in the history, like a password
-    const line = this[kIsMultiline] ? reverseString(this.history[0]) : this.history[0];
-
-    // Emit history event to notify listeners of update
-    this.emit('history', this.history);
-
-    return line;
+    return this.historyManager.addHistory(this[kIsMultiline], this[kLastCommandErrored]);
   }
 
   [kRefreshLine]() {
@@ -1172,26 +1122,12 @@ class Interface extends InterfaceConstructor {
   // <ctrl> + N. Only show this after two/three UPs or DOWNs, not on the first
   // one.
   [kHistoryNext]() {
-    if (this.historyIndex >= 0) {
-      this[kBeforeEdit](this.line, this.cursor);
-      const search = this[kSubstringSearch] || '';
-      let index = this.historyIndex - 1;
-      while (
-        index >= 0 &&
-        (!StringPrototypeStartsWith(this.history[index], search) ||
-          this.line === this.history[index])
-      ) {
-        index--;
-      }
-      if (index === -1) {
-        this[kSetLine](search);
-      } else {
-        this[kSetLine](this[kNormalizeHistoryLineEndings](this.history[index], '\r', '\n'));
-      }
-      this.historyIndex = index;
-      this.cursor = this.line.length; // Set cursor to end of line.
-      this[kRefreshLine]();
-    }
+    if (!this.historyManager.canNavigateToNext()) { return; }
+
+    this[kBeforeEdit](this.line, this.cursor);
+    this[kSetLine](this.historyManager.navigateToNext(this[kSubstringSearch]));
+    this.cursor = this.line.length; // Set cursor to end of line.
+    this[kRefreshLine]();
   }
 
   [kMoveUpOrHistoryPrev]() {
@@ -1206,26 +1142,12 @@ class Interface extends InterfaceConstructor {
   }
 
   [kHistoryPrev]() {
-    if (this.historyIndex < this.history.length && this.history.length) {
-      this[kBeforeEdit](this.line, this.cursor);
-      const search = this[kSubstringSearch] || '';
-      let index = this.historyIndex + 1;
-      while (
-        index < this.history.length &&
-        (!StringPrototypeStartsWith(this.history[index], search) ||
-          this.line === this.history[index])
-      ) {
-        index++;
-      }
-      if (index === this.history.length) {
-        this[kSetLine](search);
-      } else {
-        this[kSetLine](this[kNormalizeHistoryLineEndings](this.history[index], '\r', '\n'));
-      }
-      this.historyIndex = index;
-      this.cursor = this.line.length; // Set cursor to end of line.
-      this[kRefreshLine]();
-    }
+    if (!this.historyManager.canNavigateToPrevious()) { return; }
+
+    this[kBeforeEdit](this.line, this.cursor);
+    this[kSetLine](this.historyManager.navigateToPrevious(this[kSubstringSearch]));
+    this.cursor = this.line.length; // Set cursor to end of line.
+    this[kRefreshLine]();
   }
 
   // Returns the last character's display position of the given string

lib/internal/repl.js

@@ -47,7 +47,14 @@ function createRepl(env, opts, cb) {
     opts.historySize = 1000;
   }
 
-  const repl = REPL.start(opts);
   const term = 'terminal' in opts ? opts.terminal : process.stdout.isTTY;
-  repl.setupHistory(term ? env.NODE_REPL_HISTORY : '', cb);
+  opts.historyFile = term ? env.NODE_REPL_HISTORY : '';
+
+  const repl = REPL.start(opts);
+
+  repl.setupHistory({
+    historyFile: opts.historyFile,
+    historySize: opts.historySize,
+    onHistoryFileLoaded: cb,
+  });
 }