feat(NODE-6773): add support for $lookup with automatic encryption (#4427)

Author: nbbeeken

.gitignore

@@ -99,6 +99,7 @@ test/lambda/env.json
 
 # files generated by tooling in drivers-evergreen-tools
 secrets-export.sh
+secrets-export.fish
 mo-expansion.sh
 mo-expansion.yml
 expansions.sh

etc/bash_to_fish.mjs

@@ -0,0 +1,39 @@
+import { createReadStream, promises as fs } from 'node:fs';
+import path from 'node:path';
+import readline from 'node:readline/promises';
+
+/**
+ * Takes an "exports" only bash script file
+ * and converts it to fish syntax.
+ * Will crash on any line that isn't:
+ * - a comment
+ * - an empty line
+ * - a bash 'set' call
+ * - export VAR=VAL
+ */
+
+const fileName = process.argv[2];
+const outFileName = path.basename(fileName, '.sh') + '.fish';
+const input = createReadStream(process.argv[2]);
+const lines = readline.createInterface({ input });
+const output = await fs.open(outFileName, 'w');
+
+for await (let line of lines) {
+  line = line.trim();
+
+  if (!line.startsWith('export ')) {
+    if (line.startsWith('#')) continue;
+    if (line === '') continue;
+    if (line.startsWith('set')) continue;
+    throw new Error('Cannot translate: ' + line);
+  }
+
+  const varVal = line.slice('export '.length);
+  const variable = varVal.slice(0, varVal.indexOf('='));
+  const value = varVal.slice(varVal.indexOf('=') + 1);
+  await output.appendFile(`set -x ${variable} ${value}\n`);
+}
+
+output.close();
+input.close();
+lines.close();

package-lock.json

@@ -96,7 +96,7 @@
     "js-yaml": "^4.1.0",
     "mocha": "^10.8.2",
     "mocha-sinon": "^2.1.2",
-    "mongodb-client-encryption": "^6.2.0",
+    "mongodb-client-encryption": "^6.3.0",
     "mongodb-legacy": "^6.1.3",
     "nyc": "^15.1.0",
     "prettier": "^3.4.2",

package.json

@@ -239,6 +239,7 @@ export class AutoEncrypter {
     this._kmsProviders = options.kmsProviders || {};
 
     const mongoCryptOptions: MongoCryptOptions = {
+      enableMultipleCollinfo: true,
       cryptoCallbacks
     };
     if (options.schemaMap) {

src/client-side-encryption/auto_encrypter.ts

@@ -16,6 +16,7 @@ import { getSocks, type SocksLib } from '../deps';
 import { MongoOperationTimeoutError } from '../error';
 import { type MongoClient, type MongoClientOptions } from '../mongo_client';
 import { type Abortable } from '../mongo_types';
+import { type CollectionInfo } from '../operations/list_collections';
 import { Timeout, type TimeoutContext, TimeoutError } from '../timeout';
 import {
   addAbortListener,
@@ -205,11 +206,19 @@ export class StateMachine {
     const mongocryptdManager = executor._mongocryptdManager;
     let result: Uint8Array | null = null;
 
-    while (context.state !== MONGOCRYPT_CTX_DONE && context.state !== MONGOCRYPT_CTX_ERROR) {
+    // Typescript treats getters just like properties: Once you've tested it for equality
+    // it cannot change. Which is exactly the opposite of what we use state and status for.
+    // Every call to at least `addMongoOperationResponse` and `finalize` can change the state.
+    // These wrappers let us write code more naturally and not add compiler exceptions
+    // to conditions checks inside the state machine.
+    const getStatus = () => context.status;
+    const getState = () => context.state;
+
+    while (getState() !== MONGOCRYPT_CTX_DONE && getState() !== MONGOCRYPT_CTX_ERROR) {
       options.signal?.throwIfAborted();
-      debug(`[context#${context.id}] ${stateToString.get(context.state) || context.state}`);
+      debug(`[context#${context.id}] ${stateToString.get(getState()) || getState()}`);
 
-      switch (context.state) {
+      switch (getState()) {
         case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO: {
           const filter = deserialize(context.nextMongoOperation());
           if (!metaDataClient) {
@@ -218,22 +227,28 @@ export class StateMachine {
             );
           }
 
-          const collInfo = await this.fetchCollectionInfo(
+          const collInfoCursor = this.fetchCollectionInfo(
             metaDataClient,
             context.ns,
             filter,
             options
           );
-          if (collInfo) {
-            context.addMongoOperationResponse(collInfo);
+
+          for await (const collInfo of collInfoCursor) {
+            context.addMongoOperationResponse(serialize(collInfo));
+            if (getState() === MONGOCRYPT_CTX_ERROR) break;
           }
 
+          if (getState() === MONGOCRYPT_CTX_ERROR) break;
+
           context.finishMongoOperation();
           break;
         }
 
         case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS: {
           const command = context.nextMongoOperation();
+          if (getState() === MONGOCRYPT_CTX_ERROR) break;
+
           if (!mongocryptdClient) {
             throw new MongoCryptError(
               'unreachable state machine state: entered MONGOCRYPT_CTX_NEED_MONGO_MARKINGS but mongocryptdClient is undefined'
@@ -283,22 +298,21 @@ export class StateMachine {
 
         case MONGOCRYPT_CTX_READY: {
           const finalizedContext = context.finalize();
-          // @ts-expect-error finalize can change the state, check for error
-          if (context.state === MONGOCRYPT_CTX_ERROR) {
-            const message = context.status.message || 'Finalization error';
+          if (getState() === MONGOCRYPT_CTX_ERROR) {
+            const message = getStatus().message || 'Finalization error';
             throw new MongoCryptError(message);
           }
           result = finalizedContext;
           break;
         }
 
         default:
-          throw new MongoCryptError(`Unknown state: ${context.state}`);
+          throw new MongoCryptError(`Unknown state: ${getState()}`);
       }
     }
 
-    if (context.state === MONGOCRYPT_CTX_ERROR || result == null) {
-      const message = context.status.message;
+    if (getState() === MONGOCRYPT_CTX_ERROR || result == null) {
+      const message = getStatus().message;
       if (!message) {
         debug(
           `unidentifiable error in MongoCrypt - received an error status from \`libmongocrypt\` but received no error message.`
@@ -527,29 +541,24 @@ export class StateMachine {
    * @param filter - A filter for the listCollections command
    * @param callback - Invoked with the info of the requested collection, or with an error
    */
-  async fetchCollectionInfo(
+  fetchCollectionInfo(
     client: MongoClient,
     ns: string,
     filter: Document,
     options?: { timeoutContext?: TimeoutContext } & Abortable
-  ): Promise<Uint8Array | null> {
+  ): AsyncIterable<CollectionInfo> {
     const { db } = MongoDBCollectionNamespace.fromString(ns);
 
     const cursor = client.db(db).listCollections(filter, {
       promoteLongs: false,
       promoteValues: false,
       timeoutContext:
         options?.timeoutContext && new CursorTimeoutContext(options?.timeoutContext, Symbol()),
-      signal: options?.signal
+      signal: options?.signal,
+      nameOnly: false
     });
 
-    // There is always exactly zero or one matching documents, so this should always exhaust the cursor
-    // in a single batch.  We call `toArray()` just to be safe and ensure that the cursor is always
-    // exhausted and closed.
-    const collections = await cursor.toArray();
-
-    const info = collections.length > 0 ? serialize(collections[0]) : null;
-    return info;
+    return cursor;
   }
 
   /**