feat(cli-repl): add support for bracketed paste in REPL MONGOSH-1909 (#2328)

Bracketed paste allows us to receive a copy-pasted piece of mongosh
as a single block, rather than interpreting it line-by-line.
For now, this requires some monkey-patching of Node.js internals, so
a follow-up ticket will include work to upstream support for this
into Node.js core.

Author: addaleax

.evergreen/setup-env.sh

@@ -17,6 +17,13 @@ export MONGOSH_TEST_ONLY_MAX_LOG_FILE_COUNT=100000
 export IS_MONGOSH_EVERGREEN_CI=1
 export DEBUG="mongodb*,$DEBUG"
 
+# This is, weirdly enough, specifically set on s390x hosts, but messes
+# with our e2e tests.
+if [ x"$TERM" = x"dumb" ]; then
+  unset TERM
+fi
+echo "TERM variable is set to '${TERM:-}'"
+
 if [ "$OS" != "Windows_NT" ]; then
   if which realpath; then # No realpath on macOS, but also not needed there
     export HOME="$(realpath "$HOME")" # Needed to de-confuse nvm when /home is a symlink

package-lock.json

@@ -43,7 +43,7 @@
     "access": "public"
   },
   "engines": {
-    "node": ">=16.15.0"
+    "node": ">=18.19.0"
   },
   "mongosh": {
     "ciRequiredOptionalDependencies": {

packages/cli-repl/package.json

@@ -313,4 +313,43 @@ describe('AsyncRepl', function () {
       });
     });
   });
+
+  it('does not run pasted text immediately', async function () {
+    const { input, output } = createDefaultAsyncRepl({
+      terminal: true,
+      useColors: false,
+    });
+
+    output.read(); // Read prompt so it doesn't mess with further output
+    input.write('\x1b[200~1234\n*5678\n\x1b[201~');
+    await tick();
+    // ESC[nG is horizontal cursor movement, ESC[nJ is cursor display reset
+    expect(output.read()).to.equal(
+      '1234\r\n\x1B[1G\x1B[0J... \x1B[5G*5678\r\n\x1B[1G\x1B[0J... \x1B[5G'
+    );
+    input.write('\n');
+    await tick();
+    // Contains the expected result after hitting newline
+    expect(output.read()).to.equal('\r\n7006652\n\x1B[1G\x1B[0J> \x1B[3G');
+  });
+
+  it('allows using ctrl+c to avoid running pasted text', async function () {
+    const { input, output } = createDefaultAsyncRepl({
+      terminal: true,
+      useColors: false,
+    });
+
+    output.read(); // Read prompt so it doesn't mess with further output
+    input.write('\x1b[200~1234\n*5678\n\x1b[201~');
+    await tick();
+    expect(output.read()).to.equal(
+      '1234\r\n\x1B[1G\x1B[0J... \x1B[5G*5678\r\n\x1B[1G\x1B[0J... \x1B[5G'
+    );
+    input.write('\x03'); // Ctrl+C
+    await tick();
+    expect(output.read()).to.equal('\r\n\x1b[1G\x1b[0J> \x1b[3G');
+    input.write('"foo";\n'); // Write something else
+    await tick();
+    expect(output.read()).to.equal(`"foo";\r\n'foo'\n\x1B[1G\x1B[0J> \x1B[3G`);
+  });
 });

packages/cli-repl/src/async-repl.spec.ts

@@ -5,6 +5,7 @@ import type { ReadLineOptions } from 'readline';
 import type { ReplOptions, REPLServer } from 'repl';
 import type { start as originalStart } from 'repl';
 import { promisify } from 'util';
+import type { KeypressKey } from './repl-paste-support';
 
 // Utility, inverse of Readonly<T>
 type Mutable<T> = {
@@ -75,7 +76,9 @@ function getPrompt(repl: any): string {
 export function start(opts: AsyncREPLOptions): REPLServer {
   // 'repl' is not supported in startup snapshots yet.
   // eslint-disable-next-line @typescript-eslint/no-var-requires
-  const { Recoverable, start: originalStart } = require('repl');
+  const { Recoverable, start: originalStart } =
+    // eslint-disable-next-line @typescript-eslint/consistent-type-imports
+    require('repl') as typeof import('repl');
   const { asyncEval, wrapCallbackError = (err) => err, onAsyncSigint } = opts;
   if (onAsyncSigint) {
     (opts as ReplOptions).breakEvalOnSigint = true;
@@ -96,12 +99,28 @@ export function start(opts: AsyncREPLOptions): REPLServer {
     return wasInRawMode;
   };
 
+  // TODO(MONGOSH-1911): Upstream this feature into Node.js core.
+  let isPasting = false;
+  repl.input.on('keypress', (s: string, key: KeypressKey) => {
+    if (key.name === 'paste-start') {
+      isPasting = true;
+    } else if (key.name === 'paste-end') {
+      isPasting = false;
+    }
+  });
+
   (repl as Mutable<typeof repl>).eval = (
     input: string,
     context: any,
     filename: string,
     callback: (err: Error | null, result?: any) => void
   ): void => {
+    if (isPasting) {
+      return callback(
+        new Recoverable(new Error('recoverable because pasting in progress'))
+      );
+    }
+
     async function _eval() {
       let previouslyInRawMode;
 

packages/cli-repl/src/async-repl.ts

@@ -2612,8 +2612,7 @@ describe('CliRepl', function () {
       for (const { version, deprecated } of [
         { version: 'v20.5.1', deprecated: false },
         { version: '20.0.0', deprecated: false },
-        { version: '18.0.0', deprecated: true },
-        { version: '16.20.3', deprecated: true },
+        { version: '18.19.0', deprecated: true },
       ]) {
         delete (process as any).version;
         (process as any).version = version;
@@ -2639,7 +2638,7 @@ describe('CliRepl', function () {
 
     it('does not print any deprecation warning when CLI is ran with --quiet flag', async function () {
       // Setting all the possible situation for a deprecation warning
-      process.version = '16.20.3';
+      process.version = '18.20.0';
       process.versions.openssl = '1.1.11';
       cliRepl.getGlibcVersion = () => '1.27';
 

packages/cli-repl/src/cli-repl.spec.ts

@@ -1,5 +1,6 @@
 import { Readable } from 'stream';
 import { StringDecoder } from 'string_decoder';
+import type { ReadStream } from 'tty';
 
 const LINE_ENDING_RE = /\r?\n|\r(?!\n)/;
 const CTRL_C = '\u0003';
@@ -22,14 +23,14 @@ const CTRL_D = '\u0004';
  * the proxied `tty.ReadStream`, forwarding all the characters.
  */
 export class LineByLineInput extends Readable {
-  private _originalInput: NodeJS.ReadStream;
+  private _originalInput: Readable & Partial<ReadStream>;
   private _forwarding: boolean;
   private _blockOnNewLineEnabled: boolean;
   private _charQueue: (string | null)[];
   private _decoder: StringDecoder;
   private _insidePushCalls: number;
 
-  constructor(readable: NodeJS.ReadStream) {
+  constructor(readable: Readable & Partial<ReadStream>) {
     super();
     this._originalInput = readable;
     this._forwarding = true;
@@ -64,7 +65,7 @@ export class LineByLineInput extends Readable {
     );
 
     const proxy = new Proxy(readable, {
-      get: (target: NodeJS.ReadStream, property: string): any => {
+      get: (target: typeof readable, property: string): any => {
         if (
           typeof property === 'string' &&
           !property.startsWith('_') &&

packages/cli-repl/src/line-by-line-input.ts

@@ -46,6 +46,7 @@ import type { FormatOptions } from './format-output';
 import { markTime } from './startup-timing';
 import type { Context } from 'vm';
 import { Script, createContext, runInContext } from 'vm';
+import { installPasteSupport } from './repl-paste-support';
 
 declare const __non_webpack_require__: any;
 
@@ -135,6 +136,7 @@ class MongoshNodeRepl implements EvaluationListener {
   input: Readable;
   lineByLineInput: LineByLineInput;
   output: Writable;
+  outputFinishString = ''; // Can add ANSI escape codes to reset state from previously written ones
   bus: MongoshBus;
   nodeReplOptions: Partial<ReplOptions>;
   shellCliOptions: Partial<MongoshCliOptions>;
@@ -251,7 +253,7 @@ class MongoshNodeRepl implements EvaluationListener {
         // 'repl' is not supported in startup snapshots yet.
         // eslint-disable-next-line @typescript-eslint/no-var-requires
         start: require('pretty-repl').start,
-        input: this.lineByLineInput as unknown as Readable,
+        input: this.lineByLineInput,
         output: this.output,
         prompt: '',
         writer: this.writer.bind(this),
@@ -387,6 +389,8 @@ class MongoshNodeRepl implements EvaluationListener {
     const { repl, instanceState } = this.runtimeState();
     if (!repl) return;
 
+    this.outputFinishString += installPasteSupport(repl);
+
     const origReplCompleter = promisify(repl.completer.bind(repl)); // repl.completer is callback-style
     const mongoshCompleter = completer.bind(
       null,
@@ -1079,7 +1083,9 @@ class MongoshNodeRepl implements EvaluationListener {
         await once(rs.repl, 'exit');
       }
       await rs.instanceState.close(true);
-      await new Promise((resolve) => this.output.write('', resolve));
+      await new Promise((resolve) =>
+        this.output.write(this.outputFinishString, resolve)
+      );
     }
   }
 

packages/cli-repl/src/mongosh-repl.ts

@@ -0,0 +1,91 @@
+import type { ReplOptions, REPLServer } from 'repl';
+import { start } from 'repl';
+import type { Readable, Writable } from 'stream';
+import { PassThrough } from 'stream';
+import { tick } from '../test/repl-helpers';
+import { installPasteSupport } from './repl-paste-support';
+import { expect } from 'chai';
+
+function createTerminalRepl(extraOpts: Partial<ReplOptions> = {}): {
+  input: Writable;
+  output: Readable;
+  repl: REPLServer;
+} {
+  const input = new PassThrough();
+  const output = new PassThrough({ encoding: 'utf8' });
+
+  const repl = start({
+    input: input,
+    output: output,
+    prompt: '> ',
+    terminal: true,
+    useColors: false,
+    ...extraOpts,
+  });
+  return { input, output, repl };
+}
+
+describe('installPasteSupport', function () {
+  it('does nothing for non-terminal REPL instances', async function () {
+    const { repl, output } = createTerminalRepl({ terminal: false });
+    const onFinish = installPasteSupport(repl);
+    await tick();
+    expect(output.read()).to.equal('> ');
+    expect(onFinish).to.equal('');
+  });
+
+  it('prints a control character sequence that indicates support for bracketed paste', async function () {
+    const { repl, output } = createTerminalRepl();
+    const onFinish = installPasteSupport(repl);
+    await tick();
+    expect(output.read()).to.include('\x1B[?2004h');
+    expect(onFinish).to.include('\x1B[?2004l');
+  });
+
+  it('echoes back control characters in the input by default', async function () {
+    const { repl, input, output } = createTerminalRepl();
+    installPasteSupport(repl);
+    await tick();
+    output.read(); // Ignore prompt etc.
+    input.write('foo\x1b[Dbar'); // ESC[D = 1 character to the left
+    await tick();
+    expect(output.read()).to.equal(
+      'foo\x1B[1D\x1B[1G\x1B[0J> fobo\x1B[6G\x1B[1G\x1B[0J> fobao\x1B[7G\x1B[1G\x1B[0J> fobaro\x1B[8G'
+    );
+  });
+
+  it('ignores control characters in the input while pasting', async function () {
+    const { repl, input, output } = createTerminalRepl();
+    installPasteSupport(repl);
+    await tick();
+    output.read(); // Ignore prompt etc.
+    input.write('\x1b[200~foo\x1b[Dbar\x1b[201~'); // ESC[D = 1 character to the left
+    await tick();
+    expect(output.read()).to.equal('foobar');
+  });
+
+  it('resets to accepting control characters in the input after pasting', async function () {
+    const { repl, input, output } = createTerminalRepl();
+    installPasteSupport(repl);
+    await tick();
+    output.read();
+    input.write('\x1b[200~foo\x1b[Dbar\x1b[201~'); // ESC[D = 1 character to the left
+    await tick();
+    output.read();
+    input.write('foo\x1b[Dbar');
+    await tick();
+    expect(output.read()).to.equal(
+      'foo\x1B[1D\x1B[1G\x1B[0J> foobarfobo\x1B[12G\x1B[1G\x1B[0J> foobarfobao\x1B[13G\x1B[1G\x1B[0J> foobarfobaro\x1B[14G'
+    );
+  });
+
+  it('allows a few special characters while pasting', async function () {
+    const { repl, input, output } = createTerminalRepl();
+    installPasteSupport(repl);
+    await tick();
+    output.read();
+    input.write('\x1b[200~12*34\n_*_\n\x1b[201~');
+    await tick();
+    expect(output.read()).to.include((12 * 34) ** 2);
+  });
+});

packages/cli-repl/src/repl-paste-support.spec.ts

@@ -0,0 +1,67 @@
+import type { REPLServer } from 'repl';
+
+// https://github.com/nodejs/node/blob/d9786109b2a0982677135f0c146f6b591a0e4961/lib/internal/readline/utils.js#L90
+// https://nodejs.org/api/readline.html#readlineemitkeypresseventsstream-interface
+export type KeypressKey = {
+  sequence: string | null;
+  name: string | undefined;
+  ctrl: boolean;
+  meta: boolean;
+  shift: boolean;
+  code?: string;
+};
+
+function* prototypeChain(obj: unknown): Iterable<unknown> {
+  if (!obj) return;
+  yield obj;
+  yield* prototypeChain(Object.getPrototypeOf(obj));
+}
+
+export function installPasteSupport(repl: REPLServer): string {
+  if (!repl.terminal || process.env.TERM === 'dumb') return ''; // No paste needed in non-terminal environments
+
+  // TODO(MONGOSH-1911): Upstream as much of this into Node.js core as possible,
+  // both because of the value to the wider community but also because this is
+  // messing with Node.js REPL internals to a very unfortunate degree.
+  repl.output.write('\x1b[?2004h'); // Indicate support for paste mode
+  const onEnd = '\x1b[?2004l'; // End of support for paste mode
+  // Find the symbol used for the (internal) _ttyWrite method of readline.Interface
+  // https://github.com/nodejs/node/blob/d9786109b2a0982677135f0c146f6b591a0e4961/lib/internal/readline/interface.js#L1056
+  const ttyWriteKey = [...prototypeChain(repl)]
+    .flatMap((proto) => Object.getOwnPropertySymbols(proto))
+    .find((s) => String(s).includes('(_ttyWrite)'));
+  if (!ttyWriteKey)
+    throw new Error('Could not find _ttyWrite key on readline instance');
+  repl.input.on('keypress', (s: string, key: KeypressKey) => {
+    if (key.name === 'paste-start') {
+      if (Object.prototype.hasOwnProperty.call(repl, ttyWriteKey))
+        throw new Error(
+          'Unexpected existing own _ttyWrite key on readline instance'
+        );
+      const origTtyWrite = (repl as any)[ttyWriteKey];
+      Object.defineProperty(repl as any, ttyWriteKey, {
+        value: function (s: string, key: KeypressKey) {
+          if (key.ctrl || key.meta || key.code) {
+            // Special character or escape code sequence, ignore while pasting
+            return;
+          }
+          if (
+            key.name &&
+            key.name !== key.sequence?.toLowerCase() &&
+            !['tab', 'return', 'enter', 'space'].includes(key.name)
+          ) {
+            // Special character or escape code sequence, ignore while pasting
+            return;
+          }
+          return origTtyWrite.call(this, s, key);
+        },
+        enumerable: false,
+        writable: true,
+        configurable: true,
+      });
+    } else if (key.name === 'paste-end') {
+      delete (repl as any)[ttyWriteKey];
+    }
+  });
+  return onEnd;
+}