feat(NODE-6627): add TS support to encrypted schemas

.eslintrc.js

@@ -14,7 +14,9 @@ module.exports = {
     '**/docs/js/native.js',
     '!.*',
     'node_modules',
-    '.git'
+    '.git',
+    'data',
+    '.config'
   ],
   overrides: [
     {
@@ -104,7 +106,7 @@ module.exports = {
     // 'markdown'
   ],
   parserOptions: {
-    ecmaVersion: 2020
+    ecmaVersion: 2022
   },
   env: {
     node: true,

.github/workflows/encryption-tests.yml

@@ -0,0 +1,39 @@
+name: Encryption Tests 
+
+on:
+  push: 
+    branches: ['master']
+  pull_request:
+    branches: [ 'master' ]
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+  id-token: write
+
+jobs:
+  run-tests:
+    permissions:
+      # required for all workflows
+      security-events: write
+      id-token: write
+      contents: write
+    runs-on: ubuntu-latest
+    name: Encryption tests
+    env:
+      FORCE_COLOR: true
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Setup node
+        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        with:
+          node-version: 22
+      - name: Install Dependencies
+        run: npm install
+      - name: Install mongodb-client-encryption
+        run: npm install mongodb-client-encryption
+      - name: Setup Tests
+        run: npm run setup-test-encryption
+      - name: Run Tests
+        run: npm run test-encryption

.github/workflows/test.yml

@@ -108,7 +108,7 @@ jobs:
       - name: Setup Deno
         uses: denoland/setup-deno@v2
         with:
-          deno-version: v1.37.x
+          deno-version: v2.1.x
       - run: deno --version
       - run: npm install
       - name: Run Deno tests

.gitignore

@@ -67,3 +67,6 @@ examples/ecommerce-netlify-functions/.netlify/state.json
 
 notes.md
 list.out
+
+data
+*.pid

CONTRIBUTING.md

@@ -46,6 +46,13 @@ If you have a question about Mongoose (not a bug report) please post it to eithe
   * execute `npm run test-tsd` to run the typescript tests
   * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time.
   * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command.
+* in order to run tests that require an cluster with encryption locally, run `npm run setup-test-encryption` followed by `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file.
+  * These scripts can take a few minutes to run.
+  * To change an encryption configuration, it is recommended to follow these steps:
+    * Edit the variables in `scripts/configure-cluster-with-encryption.sh` with your desired configuration.
+    * Restart your shell.
+    * Delete the `data/` directory if it exists.
+    * Finally, run the configuration script.
 
 ## Documentation
 

docs/field-level-encryption.md

@@ -112,3 +112,41 @@ With the above connection, if you create a model named 'Test' that uses the 'tes
 const Model = mongoose.model('Test', mongoose.Schema({ name: String }));
 await Model.create({ name: 'super secret' });
 ```
+
+## Automatic FLE in Mongoose
+
+Mongoose supports the declaration of encrypted schemas - schemas that, when connected to a model, utilize MongoDB's Client Side
+Field Level Encryption or Queryable Encryption under the hood.  Mongoose automatically generates either an `encryptedFieldsMap` or a
+`schemaMap` when instantiating a MongoClient and encrypts fields on write and decrypts fields on reads.
+
+### Encryption types
+
+MongoDB has two different automatic encryption implementations: client side field level encryption (CSFLE) and queryable encryption (QE).  
+See [choosing an in-use encryption approach](https://www.mongodb.com/docs/v7.3/core/queryable-encryption/about-qe-csfle/#choosing-an-in-use-encryption-approach).
+
+###  Declaring Encrypted Schemas
+
+The following schema declares two properties, `name` and `ssn`.  `ssn` is encrypted using queryable encryption, and
+is configured for equality queries:
+
+```javascript
+const encryptedUserSchema = new Schema({ 
+  name: String,
+  ssn: { 
+    type: String, 
+    // 1
+    encrypt: { 
+      keyId: '<uuid string of key id>',
+      queries: 'equality'
+    }
+  }
+  // 2
+}, { encryptionType: 'queryableEncryption' });
+```
+
+To declare a field as encrypted, you must:
+
+1. Annotate the field with encryption metadata in the schema definition
+2. Choose an encryption type for the schema and configure the schema for the encryption type
+
+Not all schematypes are supported for CSFLE and QE.  For an overview of valid schema types, refer to MongoDB's documentation.

lib/aggregate.js

@@ -13,6 +13,7 @@ const getConstructorName = require('./helpers/getConstructorName');
 const prepareDiscriminatorPipeline = require('./helpers/aggregate/prepareDiscriminatorPipeline');
 const stringifyFunctionOperators = require('./helpers/aggregate/stringifyFunctionOperators');
 const utils = require('./utils');
+const { modelSymbol } = require('./helpers/symbols');
 const read = Query.prototype.read;
 const readConcern = Query.prototype.readConcern;
 
@@ -46,13 +47,17 @@ const validRedactStringValues = new Set(['$$DESCEND', '$$PRUNE', '$$KEEP']);
  * @see MongoDB https://www.mongodb.com/docs/manual/applications/aggregation/
  * @see driver https://mongodb.github.io/node-mongodb-native/4.9/classes/Collection.html#aggregate
  * @param {Array} [pipeline] aggregation pipeline as an array of objects
- * @param {Model} [model] the model to use with this aggregate.
+ * @param {Model|Connection} [modelOrConn] the model or connection to use with this aggregate.
  * @api public
  */
 
-function Aggregate(pipeline, model) {
+function Aggregate(pipeline, modelOrConn) {
   this._pipeline = [];
-  this._model = model;
+  if (modelOrConn == null || modelOrConn[modelSymbol]) {
+    this._model = modelOrConn;
+  } else {
+    this._connection = modelOrConn;
+  }
   this.options = {};
 
   if (arguments.length === 1 && Array.isArray(pipeline)) {
@@ -1029,12 +1034,24 @@ Aggregate.prototype.pipeline = function() {
  */
 
 Aggregate.prototype.exec = async function exec() {
-  if (!this._model) {
+  if (!this._model && !this._connection) {
     throw new Error('Aggregate not bound to any Model');
   }
   if (typeof arguments[0] === 'function') {
     throw new MongooseError('Aggregate.prototype.exec() no longer accepts a callback');
   }
+
+  if (this._connection) {
+    if (!this._pipeline.length) {
+      throw new MongooseError('Aggregate has empty pipeline');
+    }
+
+    this._optionsForExec();
+
+    const cursor = await this._connection.client.db().aggregate(this._pipeline, this.options);
+    return await cursor.toArray();
+  }
+
   const model = this._model;
   const collection = this._model.collection;
 

lib/collection.js

@@ -29,7 +29,7 @@ function Collection(name, conn, opts) {
   this.collectionName = name;
   this.conn = conn;
   this.queue = [];
-  this.buffer = true;
+  this.buffer = !conn?._hasOpened;
   this.emitter = new EventEmitter();
 
   if (STATES.connected === this.conn.readyState) {

lib/connection.js

@@ -1742,6 +1742,18 @@ Connection.prototype.syncIndexes = async function syncIndexes(options = {}) {
  * @api public
  */
 
+/**
+ * Runs a [db-level aggregate()](https://www.mongodb.com/docs/manual/reference/method/db.aggregate/) on this connection's underlying `db`
+ *
+ * @method aggregate
+ * @memberOf Connection
+ * @param {Array} pipeline
+ * @param {Object} [options]
+ * @param {Boolean} [options.cursor=false] If true, make the Aggregate resolve to a Mongoose AggregationCursor rather than an array
+ * @return {Aggregate} Aggregation wrapper
+ * @api public
+ */
+
 /**
  * Removes the database connection with the given name created with with `useDb()`.
  *

lib/cursor/aggregationCursor.js

@@ -41,11 +41,17 @@ function AggregationCursor(agg) {
   this.cursor = null;
   this.agg = agg;
   this._transforms = [];
+  const connection = agg._connection;
   const model = agg._model;
   delete agg.options.cursor.useMongooseAggCursor;
   this._mongooseOptions = {};
 
-  _init(model, this, agg);
+  if (connection) {
+    this.cursor = connection.db.aggregate(agg._pipeline, agg.options || {});
+    setImmediate(() => this.emit('cursor', this.cursor));
+  } else {
+    _init(model, this, agg);
+  }
 }
 
 util.inherits(AggregationCursor, Readable);

lib/document.js

@@ -807,8 +807,8 @@ function init(self, obj, doc, opts, prefix) {
               reason: e
             }));
           }
-        } else if (opts.hydratedPopulatedDocs) {
-          doc[i] = schemaType.cast(value, self, true);
+        } else if (schemaType && opts.hydratedPopulatedDocs) {
+          doc[i] = schemaType.cast(value, self, true, undefined, { hydratedPopulatedDocs: true });
 
           if (doc[i] && doc[i].$__ && doc[i].$__.wasPopulated) {
             self.$populated(path, doc[i].$__.wasPopulated.value, doc[i].$__.wasPopulated.options);
@@ -4256,24 +4256,25 @@ function applySchemaTypeTransforms(self, json) {
 
   for (const path of paths) {
     const schematype = schema.paths[path];
-    if (typeof schematype.options.transform === 'function') {
+    const topLevelTransformFunction = schematype.options.transform ?? schematype.constructor?.defaultOptions?.transform;
+    const embeddedSchemaTypeTransformFunction = schematype.$embeddedSchemaType?.options?.transform
+      ?? schematype.$embeddedSchemaType?.constructor?.defaultOptions?.transform;
+    if (typeof topLevelTransformFunction === 'function') {
       const val = self.$get(path);
       if (val === undefined) {
         continue;
       }
-      const transformedValue = schematype.options.transform.call(self, val);
+      const transformedValue = topLevelTransformFunction.call(self, val);
       throwErrorIfPromise(path, transformedValue);
       utils.setValue(path, transformedValue, json);
-    } else if (schematype.$embeddedSchemaType != null &&
-        typeof schematype.$embeddedSchemaType.options.transform === 'function') {
+    } else if (typeof embeddedSchemaTypeTransformFunction === 'function') {
       const val = self.$get(path);
       if (val === undefined) {
         continue;
       }
       const vals = [].concat(val);
-      const transform = schematype.$embeddedSchemaType.options.transform;
       for (let i = 0; i < vals.length; ++i) {
-        const transformedValue = transform.call(self, vals[i]);
+        const transformedValue = embeddedSchemaTypeTransformFunction.call(self, vals[i]);
         vals[i] = transformedValue;
         throwErrorIfPromise(path, transformedValue);
       }

lib/drivers/node-mongodb-native/connection.js

@@ -132,6 +132,17 @@ NativeConnection.prototype.useDb = function(name, options) {
   return newConn;
 };
 
+/**
+ * Runs a [db-level aggregate()](https://www.mongodb.com/docs/manual/reference/method/db.aggregate/) on this connection's underlying `db`
+ *
+ * @param {Array} pipeline
+ * @param {Object} [options]
+ */
+
+NativeConnection.prototype.aggregate = function aggregate(pipeline, options) {
+  return new this.base.Aggregate(null, this).append(pipeline).option(options ?? {});
+};
+
 /**
  * Removes the database connection with the given name created with `useDb()`.
  *

lib/encryptionUtils.js

@@ -0,0 +1,72 @@
+'use strict';
+
+const schemaTypes = require('./schema/index.js');
+const SchemaBigInt = require('./schema/bigint');
+const SchemaBoolean = require('./schema/boolean');
+const SchemaBuffer = require('./schema/buffer');
+const SchemaDate = require('./schema/date');
+const SchemaDecimal128 = require('./schema/decimal128');
+const SchemaDouble = require('./schema/double');
+const SchemaInt32 = require('./schema/int32');
+const SchemaObjectId = require('./schema/objectId');
+const SchemaString = require('./schema/string');
+
+/**
+ * Given a schema and a path to a field in the schema, this returns the
+ * BSON type of the field, if it can be determined.  This method specifically
+ * **only** handles BSON types that are used for CSFLE and QE - any other
+ * BSON types will return `null`. (example: MinKey and MaxKey).
+ *
+ * @param {import('.').Schema} schema
+ * @param {string} path
+ * @returns {string}
+ */
+function inferBSONType(schema, path) {
+  const type = schema.path(path);
+
+  if (type instanceof SchemaString) {
+    return 'string';
+  }
+
+  if (type instanceof SchemaInt32) {
+    return 'int';
+  }
+
+  if (type instanceof SchemaBigInt) {
+    return 'long';
+  }
+
+  if (type instanceof SchemaBoolean) {
+    return 'bool';
+  }
+
+  if (type instanceof SchemaDate) {
+    return 'date';
+  }
+
+  if (type instanceof SchemaBuffer) {
+    return 'binData';
+  }
+
+  if (type instanceof SchemaObjectId) {
+    return 'objectId';
+  }
+
+  if (type instanceof SchemaDecimal128) {
+    return 'decimal';
+  }
+
+  if (type instanceof SchemaDouble) {
+    return 'double';
+  }
+
+  if (type instanceof schemaTypes.Array) {
+    return 'array';
+  }
+
+  return null;
+}
+
+module.exports = {
+  inferBSONType
+};

lib/helpers/createJSONSchemaTypeDefinition.js

@@ -0,0 +1,24 @@
+'use strict';
+
+/**
+ * Handles creating `{ type: 'object' }` vs `{ bsonType: 'object' }` vs `{ bsonType: ['object', 'null'] }`
+ *
+ * @param {String} type
+ * @param {String} bsonType
+ * @param {Boolean} useBsonType
+ * @param {Boolean} isRequired
+ */
+
+module.exports = function createJSONSchemaTypeArray(type, bsonType, useBsonType, isRequired) {
+  if (useBsonType) {
+    if (isRequired) {
+      return { bsonType };
+    }
+    return { bsonType: [bsonType, 'null'] };
+  } else {
+    if (isRequired) {
+      return { type };
+    }
+    return { type: [type, 'null'] };
+  }
+};