From c480bd6dbbad7682a05edd99353d0bac0d0c334a Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Mon, 29 Jan 2024 12:42:12 -0500 Subject: [PATCH 01/24] feat(model): add `recompileSchema()` function to models to allow applying schema changes after compiling Fix #14296 --- lib/model.js | 28 ++++++++++++++++++++++++++++ test/model.test.js | 13 +++++++++++++ 2 files changed, 41 insertions(+) diff --git a/lib/model.js b/lib/model.js index 0ced344223a..b793218077f 100644 --- a/lib/model.js +++ b/lib/model.js @@ -4861,6 +4861,34 @@ Model.__subclass = function subclass(conn, schema, collection) { return Model; }; +/** + * Apply changes made to this model's schema after this model was compiled. + * By default, adding virtuals and other properties to a schema after the model is compiled does nothing. + * Call this function to apply virtuals and properties that were added later. + * + * #### Example: + * const schema = new mongoose.Schema({ field: String }); + * const TestModel = mongoose.model('Test', schema); + * TestModel.schema.virtual('myVirtual').get(function() { + * return this.field + ' from myVirtual'; + * }); + * const doc = new TestModel({ field: 'Hello' }); + * doc.myVirtual; // undefined + * + * TestModel.recompileSchema(); + * doc.myVirtual; // 'Hello from myVirtual' + * + * @return {Model} + * @api public + * @memberOf Model + * @static + * @method recompileSchema + */ + +Model.recompileSchema = function recompileSchema() { + this.prototype.$__setSchema(this.schema); +}; + /** * Helper for console.log. Given a model named 'MyModel', returns the string * `'Model { MyModel }'`. diff --git a/test/model.test.js b/test/model.test.js index edff29ae047..3bd48f7018d 100644 --- a/test/model.test.js +++ b/test/model.test.js @@ -7302,6 +7302,19 @@ describe('Model', function() { /First argument to `Model` constructor must be an object/ ); }); + + it('supports recompiling model with new schema additions (gh-14296)', function() { + const schema = new mongoose.Schema({ field: String }); + const TestModel = db.model('Test', schema); + TestModel.schema.virtual('myVirtual').get(function() { + return this.field + ' from myVirtual'; + }); + const doc = new TestModel({ field: 'Hello' }); + assert.strictEqual(doc.myVirtual, undefined); + + TestModel.recompileSchema(); + assert.equal(doc.myVirtual, 'Hello from myVirtual'); + }); }); From 574cca0cfa7cf1b6cadbaa0a9560220730f47378 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Mon, 5 Feb 2024 17:08:20 -0500 Subject: [PATCH 02/24] Update lib/model.js Co-authored-by: hasezoey --- lib/model.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/model.js b/lib/model.js index b793218077f..4baef568ca8 100644 --- a/lib/model.js +++ b/lib/model.js @@ -4867,6 +4867,7 @@ Model.__subclass = function subclass(conn, schema, collection) { * Call this function to apply virtuals and properties that were added later. * * #### Example: + * * const schema = new mongoose.Schema({ field: String }); * const TestModel = mongoose.model('Test', schema); * TestModel.schema.virtual('myVirtual').get(function() { From 6bd5c0733eba0c61c452df6a963430afb98693c2 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Mon, 5 Feb 2024 17:10:02 -0500 Subject: [PATCH 03/24] docs(model): correct return type for `recompileSchema()` re: code review comments --- lib/model.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/model.js b/lib/model.js index b793218077f..1cfeb3a54cd 100644 --- a/lib/model.js +++ b/lib/model.js @@ -4878,7 +4878,7 @@ Model.__subclass = function subclass(conn, schema, collection) { * TestModel.recompileSchema(); * doc.myVirtual; // 'Hello from myVirtual' * - * @return {Model} + * @return {undefined} * @api public * @memberOf Model * @static From d18878bec22e2c34feaa0076789d03e25002ca91 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Mon, 5 Feb 2024 17:12:22 -0500 Subject: [PATCH 04/24] types(model): add recompileSchema to TypeScript types --- types/models.d.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/types/models.d.ts b/types/models.d.ts index caa4ea62696..59bef72d898 100644 --- a/types/models.d.ts +++ b/types/models.d.ts @@ -711,6 +711,8 @@ declare module 'mongoose' { options?: (mongodb.ReplaceOptions & MongooseQueryOptions) | null ): QueryWithHelpers; + recompileSchema(): void; + /** Schema the model uses. */ schema: Schema; From e33247937d2ef812ca7f1aac00dafa0c261363f8 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Tue, 6 Feb 2024 16:12:39 -0500 Subject: [PATCH 05/24] docs: add tsdoc for recompileSchema() --- types/models.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/types/models.d.ts b/types/models.d.ts index 59bef72d898..1d879a7d48a 100644 --- a/types/models.d.ts +++ b/types/models.d.ts @@ -711,6 +711,7 @@ declare module 'mongoose' { options?: (mongodb.ReplaceOptions & MongooseQueryOptions) | null ): QueryWithHelpers; + /** Apply changes made to this model's schema after this model was compiled. */ recompileSchema(): void; /** Schema the model uses. */ From dc9fd615cbd8c4493dccbfe44b9192268bff79a1 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:48:35 -0500 Subject: [PATCH 06/24] begin `withSession` --- lib/connection.js | 9 +++++++++ test/connection.test.js | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/lib/connection.js b/lib/connection.js index 1eae83a037c..adc566ea957 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -443,6 +443,15 @@ Connection.prototype.createCollections = async function createCollections(option return result; }; +/** + * A wrapper for startSession() followed by session.withTransaction(fn) + */ + + Connection.prototype.withSession = async function withSession(opts, fn) { + const session = await this.startSession(opts); + return session.withTransaction(fn); + } + /** * _Requires MongoDB >= 3.6.0._ Starts a [MongoDB session](https://www.mongodb.com/docs/manual/release-notes/3.6/#client-sessions) * for benefits like causal consistency, [retryable writes](https://www.mongodb.com/docs/manual/core/retryable-writes/), diff --git a/test/connection.test.js b/test/connection.test.js index 4b5ac0493c6..823ac165417 100644 --- a/test/connection.test.js +++ b/test/connection.test.js @@ -1553,6 +1553,12 @@ describe('connections:', function() { }); assert.deepEqual(m.connections.length, 0); }); + it('should demonstrate the withSession() function (gh-14330)', async function() { + const m = new mongoose.Mongoose(); + await m.connect(/* put mongodb connection string here */); + const res = await m.connection.withSession({}, async () => { return new Promise((resolve, reject) => { return resolve('ok')})});; + assert.ok(res); + }); describe('createCollections()', function() { it('should create collections for all models on the connection with the createCollections() function (gh-13300)', async function() { const m = new mongoose.Mongoose(); From 8e1f5ef13479e5b63421e7ba4206554b77d2c06f Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:49:59 -0500 Subject: [PATCH 07/24] fix: lint --- lib/connection.js | 8 ++++---- test/connection.test.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/connection.js b/lib/connection.js index adc566ea957..482c8a390be 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -447,10 +447,10 @@ Connection.prototype.createCollections = async function createCollections(option * A wrapper for startSession() followed by session.withTransaction(fn) */ - Connection.prototype.withSession = async function withSession(opts, fn) { - const session = await this.startSession(opts); - return session.withTransaction(fn); - } +Connection.prototype.withSession = async function withSession(opts, fn) { + const session = await this.startSession(opts); + return session.withTransaction(fn); +}; /** * _Requires MongoDB >= 3.6.0._ Starts a [MongoDB session](https://www.mongodb.com/docs/manual/release-notes/3.6/#client-sessions) diff --git a/test/connection.test.js b/test/connection.test.js index 823ac165417..08c32e5de83 100644 --- a/test/connection.test.js +++ b/test/connection.test.js @@ -1556,7 +1556,7 @@ describe('connections:', function() { it('should demonstrate the withSession() function (gh-14330)', async function() { const m = new mongoose.Mongoose(); await m.connect(/* put mongodb connection string here */); - const res = await m.connection.withSession({}, async () => { return new Promise((resolve, reject) => { return resolve('ok')})});; + const res = await m.connection.withSession({}, async() => { return new Promise((resolve, reject) => { return resolve('ok');});}); assert.ok(res); }); describe('createCollections()', function() { From 9f0b13ebffc363f0738fc86c7615a08703f3bf23 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:57:21 -0500 Subject: [PATCH 08/24] Update connection.test.js --- test/connection.test.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/connection.test.js b/test/connection.test.js index 08c32e5de83..ed01f108a0e 100644 --- a/test/connection.test.js +++ b/test/connection.test.js @@ -1554,8 +1554,11 @@ describe('connections:', function() { assert.deepEqual(m.connections.length, 0); }); it('should demonstrate the withSession() function (gh-14330)', async function() { + if (!process.env.REPLICA_SET && !process.env.START_REPLICA_SET) { + this.skip(); + } const m = new mongoose.Mongoose(); - await m.connect(/* put mongodb connection string here */); + m.connect(start.uri); const res = await m.connection.withSession({}, async() => { return new Promise((resolve, reject) => { return resolve('ok');});}); assert.ok(res); }); From a1a5c5aaad62c43fba22d1f205a5bdd3c36686e0 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:58:55 -0500 Subject: [PATCH 09/24] remove unused var --- test/connection.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/connection.test.js b/test/connection.test.js index ed01f108a0e..0b195bad5f9 100644 --- a/test/connection.test.js +++ b/test/connection.test.js @@ -1559,7 +1559,7 @@ describe('connections:', function() { } const m = new mongoose.Mongoose(); m.connect(start.uri); - const res = await m.connection.withSession({}, async() => { return new Promise((resolve, reject) => { return resolve('ok');});}); + const res = await m.connection.withSession({}, async() => { return new Promise((resolve) => { return resolve('ok');});}); assert.ok(res); }); describe('createCollections()', function() { From 579ca48237a280b054ca7095b24cae7bbe3ff886 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:11:05 -0500 Subject: [PATCH 10/24] make opts optional --- lib/connection.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/connection.js b/lib/connection.js index 482c8a390be..ae3ed62512b 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -448,8 +448,13 @@ Connection.prototype.createCollections = async function createCollections(option */ Connection.prototype.withSession = async function withSession(opts, fn) { - const session = await this.startSession(opts); - return session.withTransaction(fn); + if (arguments.length === 0) { + throw new Error('Please provide a callback function'); + } + const options = arguments.length === 1 ? {} : opts; + const session = await this.startSession(options); + const cb = arguments.length === 1 ? opts : fn; + return session.withTransaction(cb); }; /** From 12e1224ae4a426e7ca09340f349afb72d24d6f88 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Thu, 8 Feb 2024 12:26:28 -0500 Subject: [PATCH 11/24] fix: quick fixes for withSession --- lib/connection.js | 22 +++++++++++++++------- test/connection.test.js | 7 +++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/connection.js b/lib/connection.js index ae3ed62512b..542bdf3c572 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -444,17 +444,25 @@ Connection.prototype.createCollections = async function createCollections(option }; /** - * A wrapper for startSession() followed by session.withTransaction(fn) + * A convenience wrapper for `connection.client.withSession()`. + * + * #### Example: + * + * await conn.withSession(async session => { + * const doc = await TestModel.findOne().session(session); + * }); + * + * @method withSession + * @param {Function} executor called with 1 argument: a `ClientSession` instance + * @return {Promise} resolves to the return value of the executor function + * @api public */ -Connection.prototype.withSession = async function withSession(opts, fn) { +Connection.prototype.withSession = async function withSession(executor) { if (arguments.length === 0) { - throw new Error('Please provide a callback function'); + throw new Error('Please provide an executor function'); } - const options = arguments.length === 1 ? {} : opts; - const session = await this.startSession(options); - const cb = arguments.length === 1 ? opts : fn; - return session.withTransaction(cb); + return await this.client.withSession(executor); }; /** diff --git a/test/connection.test.js b/test/connection.test.js index 0b195bad5f9..9ea81e356d0 100644 --- a/test/connection.test.js +++ b/test/connection.test.js @@ -1559,8 +1559,11 @@ describe('connections:', function() { } const m = new mongoose.Mongoose(); m.connect(start.uri); - const res = await m.connection.withSession({}, async() => { return new Promise((resolve) => { return resolve('ok');});}); - assert.ok(res); + let session = null; + await m.connection.withSession(s => { + session = s; + }); + assert.ok(session); }); describe('createCollections()', function() { it('should create collections for all models on the connection with the createCollections() function (gh-13300)', async function() { From ef5e09e4d4f027a4cb8528e0ca6f8e6fc4593bba Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Thu, 8 Feb 2024 12:28:42 -0500 Subject: [PATCH 12/24] types(connection): add withSession() --- types/connection.d.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/types/connection.d.ts b/types/connection.d.ts index 35714b13c8d..b2812d01cf6 100644 --- a/types/connection.d.ts +++ b/types/connection.d.ts @@ -240,6 +240,8 @@ declare module 'mongoose' { /** Watches the entire underlying database for changes. Similar to [`Model.watch()`](/docs/api/model.html#model_Model-watch). */ watch(pipeline?: Array, options?: mongodb.ChangeStreamOptions): mongodb.ChangeStream; + + withSession(executor: (session: ClientSession) => Promise): T; } } From 491be54c0282f8f51d8df026d99f0c2ed0189e44 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Mon, 12 Feb 2024 11:23:01 -0500 Subject: [PATCH 13/24] Update model.hydrate.test.js --- test/model.hydrate.test.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/model.hydrate.test.js b/test/model.hydrate.test.js index 6cdc333ce8d..cabdf7d225f 100644 --- a/test/model.hydrate.test.js +++ b/test/model.hydrate.test.js @@ -99,5 +99,23 @@ describe('model', function() { assert.equal(hydrated.test, 'test'); assert.deepEqual(hydrated.schema.tree, C.schema.tree); }); + it('should deeply hydrate the document with the `hydratedPopulatedDocs` option (gh-4727)', async function() { + const userSchema = new Schema({ + name: String + }); + const companySchema = new Schema({ + name: String, + users: [{ ref: 'User', type: Schema.Types.ObjectId }] + }); + + const User = db.model('UserTestHydrate', userSchema); + const Company = db.model('CompanyTestHyrdrate', companySchema); + + const users = [{ _id: new Schema.ObjectId(), name: 'Val'}]; + const company = { _id: new Schema.ObjectId(), name: 'Booster', users: [users[0]] }; + + const C = Company.hydrate(company, null, { hydratedPopulatedDocs: true }); + assert.equal(C.users[0].name, 'Val'); + }); }); }); From eb449c489bebfebd1760d800c4cf92392a4ffe1c Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Mon, 12 Feb 2024 11:26:20 -0500 Subject: [PATCH 14/24] fix id generation --- test/model.hydrate.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/model.hydrate.test.js b/test/model.hydrate.test.js index cabdf7d225f..51358f08aa9 100644 --- a/test/model.hydrate.test.js +++ b/test/model.hydrate.test.js @@ -111,8 +111,8 @@ describe('model', function() { const User = db.model('UserTestHydrate', userSchema); const Company = db.model('CompanyTestHyrdrate', companySchema); - const users = [{ _id: new Schema.ObjectId(), name: 'Val'}]; - const company = { _id: new Schema.ObjectId(), name: 'Booster', users: [users[0]] }; + const users = [{ _id: new mongoose.Types.ObjectId(), name: 'Val'}]; + const company = { _id: new mongoose.Types.ObjectId(), name: 'Booster', users: [users[0]] }; const C = Company.hydrate(company, null, { hydratedPopulatedDocs: true }); assert.equal(C.users[0].name, 'Val'); From e80cff1d33e10860e1502e9c1577342929f9bc42 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Mon, 12 Feb 2024 16:52:54 -0500 Subject: [PATCH 15/24] progress --- lib/document.js | 3 --- lib/model.js | 2 +- test/model.hydrate.test.js | 1 + 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/document.js b/lib/document.js index 731be3301c7..870a20261d1 100644 --- a/lib/document.js +++ b/lib/document.js @@ -694,7 +694,6 @@ Document.prototype.$__init = function(doc, opts) { init(this, doc, this._doc, opts); markArraySubdocsPopulated(this, opts.populated); - this.$emit('init', this); this.constructor.emit('init', this); @@ -703,7 +702,6 @@ Document.prototype.$__init = function(doc, opts) { null; applyDefaults(this, this.$__.selected, this.$__.exclude, hasIncludedChildren, false, this.$__.skipDefaults); - return this; }; @@ -746,7 +744,6 @@ function init(self, obj, doc, opts, prefix) { } path = prefix + i; schemaType = docSchema.path(path); - // Should still work if not a model-level discriminator, but should not be // necessary. This is *only* to catch the case where we queried using the // base model and the discriminated model has a projection diff --git a/lib/model.js b/lib/model.js index 93e34c2eb33..a596c864833 100644 --- a/lib/model.js +++ b/lib/model.js @@ -3830,6 +3830,7 @@ Model.buildBulkWriteOperations = function buildBulkWriteOperations(documents, op * @param {Object|String|String[]} [projection] optional projection containing which fields should be selected for this document * @param {Object} [options] optional options * @param {Boolean} [options.setters=false] if true, apply schema setters when hydrating + * @param {Boolean} [options.hydratedPopulatedDocs=false] if true, populates the docs if passing pre-populated data * @return {Document} document instance * @api public */ @@ -3843,7 +3844,6 @@ Model.hydrate = function(obj, projection, options) { } obj = applyProjection(obj, projection); } - const document = require('./queryHelpers').createModel(this, obj, projection); document.$init(obj, options); return document; diff --git a/test/model.hydrate.test.js b/test/model.hydrate.test.js index 51358f08aa9..9e4e9ce3d40 100644 --- a/test/model.hydrate.test.js +++ b/test/model.hydrate.test.js @@ -115,6 +115,7 @@ describe('model', function() { const company = { _id: new mongoose.Types.ObjectId(), name: 'Booster', users: [users[0]] }; const C = Company.hydrate(company, null, { hydratedPopulatedDocs: true }); + console.log('what is C', C); assert.equal(C.users[0].name, 'Val'); }); }); From 57541cbefa33aaecd7b391e2f4d1b6d44805c7d3 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Tue, 13 Feb 2024 10:46:09 -0500 Subject: [PATCH 16/24] refactor: make query helpers more consistently fit our loose definition of helpers --- lib/aggregate.js | 4 +- lib/constants.js | 36 +++++++++++++ lib/helpers/model/applyStaticHooks.js | 2 +- lib/helpers/query/applyGlobalOption.js | 18 +++---- lib/helpers/query/applyQueryMiddleware.js | 54 -------------------- lib/helpers/query/castFilterPath.js | 3 +- lib/helpers/query/completeMany.js | 36 ------------- lib/helpers/query/validOps.js | 19 +------ lib/model.js | 38 +++++++++++++- lib/query.js | 61 +++++++++++++++++------ lib/schema.js | 3 +- 11 files changed, 133 insertions(+), 141 deletions(-) create mode 100644 lib/constants.js delete mode 100644 lib/helpers/query/applyQueryMiddleware.js delete mode 100644 lib/helpers/query/completeMany.js diff --git a/lib/aggregate.js b/lib/aggregate.js index e61daa8a5df..827f1642a60 100644 --- a/lib/aggregate.js +++ b/lib/aggregate.js @@ -1019,8 +1019,8 @@ Aggregate.prototype.exec = async function exec() { const model = this._model; const collection = this._model.collection; - applyGlobalMaxTimeMS(this.options, model); - applyGlobalDiskUse(this.options, model); + applyGlobalMaxTimeMS(this.options, model.db.options, model.base.options); + applyGlobalDiskUse(this.options, model.db.options, model.base.options); if (this.options && this.options.cursor) { return new AggregationCursor(this); diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 00000000000..83a66832b55 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,36 @@ +'use strict'; + +/*! + * ignore + */ + +const queryOperations = Object.freeze([ + // Read + 'countDocuments', + 'distinct', + 'estimatedDocumentCount', + 'find', + 'findOne', + // Update + 'findOneAndReplace', + 'findOneAndUpdate', + 'replaceOne', + 'updateMany', + 'updateOne', + // Delete + 'deleteMany', + 'deleteOne', + 'findOneAndDelete' +]); + +exports.queryOperations = queryOperations; + +/*! + * ignore + */ + +const queryMiddlewareFunctions = queryOperations.concat([ + 'validate' +]); + +exports.queryMiddlewareFunctions = queryMiddlewareFunctions; diff --git a/lib/helpers/model/applyStaticHooks.js b/lib/helpers/model/applyStaticHooks.js index 934f9452ade..957e94f2288 100644 --- a/lib/helpers/model/applyStaticHooks.js +++ b/lib/helpers/model/applyStaticHooks.js @@ -1,6 +1,6 @@ 'use strict'; -const middlewareFunctions = require('../query/applyQueryMiddleware').middlewareFunctions; +const middlewareFunctions = require('../../constants').queryMiddlewareFunctions; const promiseOrCallback = require('../promiseOrCallback'); module.exports = function applyStaticHooks(model, hooks, statics) { diff --git a/lib/helpers/query/applyGlobalOption.js b/lib/helpers/query/applyGlobalOption.js index 8888e368b9e..e93fa73d492 100644 --- a/lib/helpers/query/applyGlobalOption.js +++ b/lib/helpers/query/applyGlobalOption.js @@ -2,12 +2,12 @@ const utils = require('../../utils'); -function applyGlobalMaxTimeMS(options, model) { - applyGlobalOption(options, model, 'maxTimeMS'); +function applyGlobalMaxTimeMS(options, connectionOptions, baseOptions) { + applyGlobalOption(options, connectionOptions, baseOptions, 'maxTimeMS'); } -function applyGlobalDiskUse(options, model) { - applyGlobalOption(options, model, 'allowDiskUse'); +function applyGlobalDiskUse(options, connectionOptions, baseOptions) { + applyGlobalOption(options, connectionOptions, baseOptions, 'allowDiskUse'); } module.exports = { @@ -16,14 +16,14 @@ module.exports = { }; -function applyGlobalOption(options, model, optionName) { +function applyGlobalOption(options, connectionOptions, baseOptions, optionName) { if (utils.hasUserDefinedProperty(options, optionName)) { return; } - if (utils.hasUserDefinedProperty(model.db.options, optionName)) { - options[optionName] = model.db.options[optionName]; - } else if (utils.hasUserDefinedProperty(model.base.options, optionName)) { - options[optionName] = model.base.options[optionName]; + if (utils.hasUserDefinedProperty(connectionOptions, optionName)) { + options[optionName] = connectionOptions[optionName]; + } else if (utils.hasUserDefinedProperty(baseOptions, optionName)) { + options[optionName] = baseOptions[optionName]; } } diff --git a/lib/helpers/query/applyQueryMiddleware.js b/lib/helpers/query/applyQueryMiddleware.js deleted file mode 100644 index b3cd8ee0163..00000000000 --- a/lib/helpers/query/applyQueryMiddleware.js +++ /dev/null @@ -1,54 +0,0 @@ -'use strict'; - -/*! - * ignore - */ - -module.exports = applyQueryMiddleware; - -const validOps = require('./validOps'); - -/*! - * ignore - */ - -applyQueryMiddleware.middlewareFunctions = validOps.concat([ - 'validate' -]); - -/** - * Apply query middleware - * - * @param {Query} Query constructor - * @param {Model} model - * @api private - */ - -function applyQueryMiddleware(Query, model) { - const queryMiddleware = model.schema.s.hooks.filter(hook => { - const contexts = _getContexts(hook); - if (hook.name === 'validate') { - return !!contexts.query; - } - if (hook.name === 'deleteOne' || hook.name === 'updateOne') { - return !!contexts.query || Object.keys(contexts).length === 0; - } - if (hook.query != null || hook.document != null) { - return !!hook.query; - } - return true; - }); - - Query.prototype._queryMiddleware = queryMiddleware; -} - -function _getContexts(hook) { - const ret = {}; - if (hook.hasOwnProperty('query')) { - ret.query = hook.query; - } - if (hook.hasOwnProperty('document')) { - ret.document = hook.document; - } - return ret; -} diff --git a/lib/helpers/query/castFilterPath.js b/lib/helpers/query/castFilterPath.js index dbd0109c915..c5c8d0fadfd 100644 --- a/lib/helpers/query/castFilterPath.js +++ b/lib/helpers/query/castFilterPath.js @@ -2,8 +2,7 @@ const isOperator = require('./isOperator'); -module.exports = function castFilterPath(query, schematype, val) { - const ctx = query; +module.exports = function castFilterPath(ctx, schematype, val) { const any$conditionals = Object.keys(val).some(isOperator); if (!any$conditionals) { diff --git a/lib/helpers/query/completeMany.js b/lib/helpers/query/completeMany.js deleted file mode 100644 index a27ecafa385..00000000000 --- a/lib/helpers/query/completeMany.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict'; - -const helpers = require('../../queryHelpers'); - -module.exports = completeMany; - -/** - * Given a model and an array of docs, hydrates all the docs to be instances - * of the model. Used to initialize docs returned from the db from `find()` - * - * @param {Model} model - * @param {Array} docs - * @param {Object} fields the projection used, including `select` from schemas - * @param {Object} userProvidedFields the user-specified projection - * @param {Object} [opts] - * @param {Array} [opts.populated] - * @param {ClientSession} [opts.session] - * @param {Function} callback - * @api private - */ - -async function completeMany(model, docs, fields, userProvidedFields, opts) { - return Promise.all(docs.map(doc => new Promise((resolve, reject) => { - const rawDoc = doc; - doc = helpers.createModel(model, doc, fields, userProvidedFields); - if (opts.session != null) { - doc.$session(opts.session); - } - doc.$init(rawDoc, opts, (err) => { - if (err != null) { - return reject(err); - } - resolve(doc); - }); - }))); -} diff --git a/lib/helpers/query/validOps.js b/lib/helpers/query/validOps.js index 2eb4375a93f..2d1aa477b1c 100644 --- a/lib/helpers/query/validOps.js +++ b/lib/helpers/query/validOps.js @@ -1,20 +1,3 @@ 'use strict'; -module.exports = Object.freeze([ - // Read - 'countDocuments', - 'distinct', - 'estimatedDocumentCount', - 'find', - 'findOne', - // Update - 'findOneAndReplace', - 'findOneAndUpdate', - 'replaceOne', - 'updateMany', - 'updateOne', - // Delete - 'deleteMany', - 'deleteOne', - 'findOneAndDelete' -]); +module.exports = require('../../constants').queryMiddlewareFunctions; diff --git a/lib/model.js b/lib/model.js index 1ffca2b1b6e..8795e2ac351 100644 --- a/lib/model.js +++ b/lib/model.js @@ -22,7 +22,6 @@ const VersionError = require('./error/version'); const ParallelSaveError = require('./error/parallelSave'); const applyDefaultsHelper = require('./helpers/document/applyDefaults'); const applyDefaultsToPOJO = require('./helpers/model/applyDefaultsToPOJO'); -const applyQueryMiddleware = require('./helpers/query/applyQueryMiddleware'); const applyHooks = require('./helpers/model/applyHooks'); const applyMethods = require('./helpers/model/applyMethods'); const applyProjection = require('./helpers/projection/applyProjection'); @@ -4765,7 +4764,7 @@ Model.compile = function compile(name, schema, collectionName, connection, base) Object.setPrototypeOf(model.Query.prototype, Query.prototype); model.Query.base = Query.base; model.Query.prototype.constructor = Query; - applyQueryMiddleware(model.Query, model); + model._applyQueryMiddleware(); applyQueryMethods(model, schema.query); return model; @@ -4874,6 +4873,41 @@ if (util.inspect.custom) { Model[util.inspect.custom] = Model.inspect; } +/*! + * Applies query middleware from this model's schema to this model's + * Query constructor. + */ + +Model._applyQueryMiddleware = function _applyQueryMiddleware() { + const Query = this.Query; + const queryMiddleware = this.schema.s.hooks.filter(hook => { + const contexts = _getContexts(hook); + if (hook.name === 'validate') { + return !!contexts.query; + } + if (hook.name === 'deleteOne' || hook.name === 'updateOne') { + return !!contexts.query || Object.keys(contexts).length === 0; + } + if (hook.query != null || hook.document != null) { + return !!hook.query; + } + return true; + }); + + Query.prototype._queryMiddleware = queryMiddleware; +}; + +function _getContexts(hook) { + const ret = {}; + if (hook.hasOwnProperty('query')) { + ret.query = hook.query; + } + if (hook.hasOwnProperty('document')) { + ret.document = hook.document; + } + return ret; +} + /*! * Module exports. */ diff --git a/lib/query.js b/lib/query.js index f94acc7f0a6..245c3d48d49 100644 --- a/lib/query.js +++ b/lib/query.js @@ -19,7 +19,6 @@ const castArrayFilters = require('./helpers/update/castArrayFilters'); const castNumber = require('./cast/number'); const castUpdate = require('./helpers/query/castUpdate'); const clone = require('./helpers/clone'); -const completeMany = require('./helpers/query/completeMany'); const getDiscriminatorByValue = require('./helpers/discriminator/getDiscriminatorByValue'); const helpers = require('./queryHelpers'); const immediate = require('./helpers/immediate'); @@ -40,7 +39,7 @@ const specialProperties = require('./helpers/specialProperties'); const updateValidators = require('./helpers/updateValidators'); const util = require('util'); const utils = require('./utils'); -const validOps = require('./helpers/query/validOps'); +const queryMiddlewareFunctions = require('./constants').queryMiddlewareFunctions; const queryOptionMethods = new Set([ 'allowDiskUse', @@ -457,7 +456,7 @@ Query.prototype.slice = function() { * ignore */ -const validOpsSet = new Set(validOps); +const validOpsSet = new Set(queryMiddlewareFunctions); Query.prototype._validateOp = function() { if (this.op != null && !validOpsSet.has(this.op)) { @@ -2233,8 +2232,8 @@ Query.prototype._find = async function _find() { const _this = this; const userProvidedFields = _this._userProvidedFields || {}; - applyGlobalMaxTimeMS(this.options, this.model); - applyGlobalDiskUse(this.options, this.model); + applyGlobalMaxTimeMS(this.options, this.model.db.options, this.model.base.options); + applyGlobalDiskUse(this.options, this.model.db.options, this.model.base.options); // Separate options to pass down to `completeMany()` in case we need to // set a session on the document @@ -2271,7 +2270,7 @@ Query.prototype._find = async function _find() { } return mongooseOptions.lean ? _completeManyLean(_this.model.schema, docs, null, completeManyOptions) : - completeMany(_this.model, docs, fields, userProvidedFields, completeManyOptions); + _this._completeMany(docs, fields, userProvidedFields, completeManyOptions); } const pop = helpers.preparePopulationOptionsMQ(_this, mongooseOptions); @@ -2279,7 +2278,7 @@ Query.prototype._find = async function _find() { return _this.model.populate(docs, pop); } - docs = await completeMany(_this.model, docs, fields, userProvidedFields, completeManyOptions); + docs = await _this._completeMany(docs, fields, userProvidedFields, completeManyOptions); await this.model.populate(docs, pop); return docs; @@ -2473,6 +2472,38 @@ Query.prototype._completeOne = function(doc, res, callback) { }); }; +/** + * Given a model and an array of docs, hydrates all the docs to be instances + * of the model. Used to initialize docs returned from the db from `find()` + * + * @param {Model} model + * @param {Array} docs + * @param {Object} fields the projection used, including `select` from schemas + * @param {Object} userProvidedFields the user-specified projection + * @param {Object} [opts] + * @param {Array} [opts.populated] + * @param {ClientSession} [opts.session] + * @param {Function} callback + * @api private + */ + +Query.prototype._completeMany = async function _completeMany(docs, fields, userProvidedFields, opts) { + const model = this.model; + return Promise.all(docs.map(doc => new Promise((resolve, reject) => { + const rawDoc = doc; + doc = helpers.createModel(model, doc, fields, userProvidedFields); + if (opts.session != null) { + doc.$session(opts.session); + } + doc.$init(rawDoc, opts, (err) => { + if (err != null) { + return reject(err); + } + resolve(doc); + }); + }))); +}; + /** * Internal helper to execute a findOne() operation * @@ -2488,8 +2519,8 @@ Query.prototype._findOne = async function _findOne() { throw err; } - applyGlobalMaxTimeMS(this.options, this.model); - applyGlobalDiskUse(this.options, this.model); + applyGlobalMaxTimeMS(this.options, this.model.db.options, this.model.base.options); + applyGlobalDiskUse(this.options, this.model.db.options, this.model.base.options); const options = this._optionsForExec(); @@ -2585,8 +2616,8 @@ Query.prototype._countDocuments = async function _countDocuments() { throw this.error(); } - applyGlobalMaxTimeMS(this.options, this.model); - applyGlobalDiskUse(this.options, this.model); + applyGlobalMaxTimeMS(this.options, this.model.db.options, this.model.base.options); + applyGlobalDiskUse(this.options, this.model.db.options, this.model.base.options); const options = this._optionsForExec(); @@ -2754,8 +2785,8 @@ Query.prototype.__distinct = async function __distinct() { throw this.error(); } - applyGlobalMaxTimeMS(this.options, this.model); - applyGlobalDiskUse(this.options, this.model); + applyGlobalMaxTimeMS(this.options, this.model.db.options, this.model.base.options); + applyGlobalDiskUse(this.options, this.model.db.options, this.model.base.options); const options = this._optionsForExec(); this._applyTranslateAliases(options); @@ -3247,8 +3278,8 @@ Query.prototype._findOneAndUpdate = async function _findOneAndUpdate() { throw this.error(); } - applyGlobalMaxTimeMS(this.options, this.model); - applyGlobalDiskUse(this.options, this.model); + applyGlobalMaxTimeMS(this.options, this.model.db.options, this.model.base.options); + applyGlobalDiskUse(this.options, this.model.db.options, this.model.base.options); if ('strict' in this.options) { this._mongooseOptions.strict = this.options.strict; diff --git a/lib/schema.js b/lib/schema.js index 6f25a00ccbe..b343fbce5af 100644 --- a/lib/schema.js +++ b/lib/schema.js @@ -29,8 +29,7 @@ const hasNumericSubpathRegex = /\.\d+(\.|$)/; let MongooseTypes; -const queryHooks = require('./helpers/query/applyQueryMiddleware'). - middlewareFunctions; +const queryHooks = require('./constants').queryMiddlewareFunctions; const documentHooks = require('./helpers/model/applyHooks').middlewareFunctions; const hookNames = queryHooks.concat(documentHooks). reduce((s, hook) => s.add(hook), new Set()); From e6769c9893259a714f6fe92a6452ef428b191b4f Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:44:43 -0500 Subject: [PATCH 17/24] `hydratedPopulatedDocs` option --- lib/document.js | 5 ++--- test/model.hydrate.test.js | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/document.js b/lib/document.js index 870a20261d1..b9f39a82c12 100644 --- a/lib/document.js +++ b/lib/document.js @@ -767,15 +767,14 @@ function init(self, obj, doc, opts, prefix) { } } else { // Retain order when overwriting defaults - if (doc.hasOwnProperty(i) && obj[i] !== void 0) { + if (doc.hasOwnProperty(i) && obj[i] !== void 0 && !opts.hydratedPopulatedDocs) { delete doc[i]; } if (obj[i] === null) { doc[i] = schemaType._castNullish(null); } else if (obj[i] !== undefined) { const wasPopulated = obj[i].$__ == null ? null : obj[i].$__.wasPopulated; - - if (schemaType && !wasPopulated) { + if ((schemaType && !wasPopulated) && !opts.hydratedPopulatedDocs) { try { if (opts && opts.setters) { // Call applySetters with `init = false` because otherwise setters are a noop diff --git a/test/model.hydrate.test.js b/test/model.hydrate.test.js index 9e4e9ce3d40..51358f08aa9 100644 --- a/test/model.hydrate.test.js +++ b/test/model.hydrate.test.js @@ -115,7 +115,6 @@ describe('model', function() { const company = { _id: new mongoose.Types.ObjectId(), name: 'Booster', users: [users[0]] }; const C = Company.hydrate(company, null, { hydratedPopulatedDocs: true }); - console.log('what is C', C); assert.equal(C.users[0].name, 'Val'); }); }); From 7c696a3c92366b67de6c49704253902ea6281283 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:46:13 -0500 Subject: [PATCH 18/24] fix: lint --- test/model.hydrate.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/model.hydrate.test.js b/test/model.hydrate.test.js index 51358f08aa9..bc6632f5b15 100644 --- a/test/model.hydrate.test.js +++ b/test/model.hydrate.test.js @@ -108,10 +108,10 @@ describe('model', function() { users: [{ ref: 'User', type: Schema.Types.ObjectId }] }); - const User = db.model('UserTestHydrate', userSchema); + db.model('UserTestHydrate', userSchema); const Company = db.model('CompanyTestHyrdrate', companySchema); - const users = [{ _id: new mongoose.Types.ObjectId(), name: 'Val'}]; + const users = [{ _id: new mongoose.Types.ObjectId(), name: 'Val' }]; const company = { _id: new mongoose.Types.ObjectId(), name: 'Booster', users: [users[0]] }; const C = Company.hydrate(company, null, { hydratedPopulatedDocs: true }); From 648239d7853bffe134a213e9bc5140acb5c58b45 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 14 Feb 2024 12:10:03 -0500 Subject: [PATCH 19/24] add typescript types, begin typescript test --- test/types/models.test.ts | 22 ++++++++++++++++++++++ types/models.d.ts | 7 ++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/test/types/models.test.ts b/test/types/models.test.ts index 9bf423f136f..dd92a46f602 100644 --- a/test/types/models.test.ts +++ b/test/types/models.test.ts @@ -880,3 +880,25 @@ async function gh13999() { } } } + +function gh4727() { + const userSchema = new mongoose.Schema({ + name: String + }); + const companySchema = new mongoose.Schema({ + name: String, + users: [{ ref: 'User', type: mongoose.Schema.Types.ObjectId }] + }); + + mongoose.model('UserTestHydrate', userSchema); + const Company = mongoose.model('CompanyTestHyrdrate', companySchema); + + const users = [{ _id: new mongoose.Types.ObjectId(), name: 'Val' }]; + const company = { _id: new mongoose.Types.ObjectId(), name: 'Booster', users: [users[0]] }; + + expectType | null>( + Company.hydrate(company, null, { hydratedPopulatedDocs: true }) + ); + + +} diff --git a/types/models.d.ts b/types/models.d.ts index 67731cc3eb0..a77aa25525e 100644 --- a/types/models.d.ts +++ b/types/models.d.ts @@ -37,6 +37,11 @@ declare module 'mongoose' { skipValidation?: boolean; } + interface HydrateOptions { + setters?: boolean; + hydratedPopulatedDocs?: boolean; + } + interface InsertManyOptions extends PopulateOption, SessionOption { @@ -371,7 +376,7 @@ declare module 'mongoose' { * Shortcut for creating a new Document from existing raw data, pre-saved in the DB. * The document returned has no paths marked as modified initially. */ - hydrate(obj: any, projection?: AnyObject, options?: { setters?: boolean }): THydratedDocumentType; + hydrate(obj: any, projection?: AnyObject, options?: HydrateOptions): THydratedDocumentType; /** * This function is responsible for building [indexes](https://www.mongodb.com/docs/manual/indexes/), From 46756713ffde1f0bbfe09f9de50c66019c5c9029 Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:33:43 -0500 Subject: [PATCH 20/24] Update models.test.ts --- test/types/models.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/types/models.test.ts b/test/types/models.test.ts index dd92a46f602..6761f031456 100644 --- a/test/types/models.test.ts +++ b/test/types/models.test.ts @@ -896,9 +896,7 @@ function gh4727() { const users = [{ _id: new mongoose.Types.ObjectId(), name: 'Val' }]; const company = { _id: new mongoose.Types.ObjectId(), name: 'Booster', users: [users[0]] }; - expectType | null>( - Company.hydrate(company, null, { hydratedPopulatedDocs: true }) - ); + return Company.hydrate(company, {}, { hydratedPopulatedDocs: true }) } From 6dafa54f57341f44af13ff0451decf435cd1845d Mon Sep 17 00:00:00 2001 From: Daniel Diaz <39510674+IslandRhythms@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:34:21 -0500 Subject: [PATCH 21/24] fix: lint --- test/types/models.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/types/models.test.ts b/test/types/models.test.ts index 6761f031456..61298be836d 100644 --- a/test/types/models.test.ts +++ b/test/types/models.test.ts @@ -896,7 +896,7 @@ function gh4727() { const users = [{ _id: new mongoose.Types.ObjectId(), name: 'Val' }]; const company = { _id: new mongoose.Types.ObjectId(), name: 'Booster', users: [users[0]] }; - return Company.hydrate(company, {}, { hydratedPopulatedDocs: true }) + return Company.hydrate(company, {}, { hydratedPopulatedDocs: true }); } From aa6564526e784564f5f3409cdfb7d1ea29eefd72 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Mon, 19 Feb 2024 14:51:55 -0500 Subject: [PATCH 22/24] feat: add middleware for bulkWrite() and createCollection() Fix #7893 Fix #14263 --- docs/middleware.md | 4 + lib/model.js | 185 +++++++++++++++++++++++----------- test/model.middleware.test.js | 133 ++++++++++++++++++++++++ test/types/middleware.test.ts | 9 ++ types/index.d.ts | 42 +++++++- 5 files changed, 312 insertions(+), 61 deletions(-) diff --git a/docs/middleware.md b/docs/middleware.md index 923379ff9f2..f7c8c0bb4ff 100644 --- a/docs/middleware.md +++ b/docs/middleware.md @@ -67,13 +67,17 @@ Model middleware is supported for the following model functions. Don't confuse model middleware and document middleware: model middleware hooks into *static* functions on a `Model` class, document middleware hooks into *methods* on a `Model` class. In model middleware functions, `this` refers to the model. +* [bulkWrite](api/model.html#model_Model-bulkWrite) +* [createCollection](api/model.html#model_Model-createCollection) * [insertMany](api/model.html#model_Model-insertMany) Here are the possible strings that can be passed to `pre()` * aggregate +* bulkWrite * count * countDocuments +* createCollection * deleteOne * deleteMany * estimatedDocumentCount diff --git a/lib/model.js b/lib/model.js index 0ced344223a..e4754aa1e91 100644 --- a/lib/model.js +++ b/lib/model.js @@ -1415,6 +1415,18 @@ Model.createCollection = async function createCollection(options) { throw new MongooseError('Model.createCollection() no longer accepts a callback'); } + const shouldSkip = await new Promise((resolve, reject) => { + this.hooks.execPre('createCollection', this, [options], (err) => { + if (err != null) { + if (err instanceof Kareem.skipWrappedFunction) { + return resolve(true); + } + return reject(err); + } + resolve(); + }); + }); + const collectionOptions = this && this.schema && this.schema.options && @@ -1468,13 +1480,32 @@ Model.createCollection = async function createCollection(options) { } try { - await this.db.createCollection(this.$__collection.collectionName, options); + if (!shouldSkip) { + await this.db.createCollection(this.$__collection.collectionName, options); + } } catch (err) { - if (err != null && (err.name !== 'MongoServerError' || err.code !== 48)) { - throw err; + await new Promise((resolve, reject) => { + const _opts = { error: err }; + this.hooks.execPost('createCollection', this, [null], _opts, (err) => { + if (err != null) { + return reject(err); + } + resolve(); + }); + }); } } + + await new Promise((resolve, reject) => { + this.hooks.execPost('createCollection', this, [this.$__collection], (err) => { + if (err != null) { + return reject(err); + } + resolve(); + }); + }); + return this.$__collection; }; @@ -3428,44 +3459,62 @@ Model.bulkWrite = async function bulkWrite(ops, options) { throw new MongooseError('Model.bulkWrite() no longer accepts a callback'); } options = options || {}; + + const shouldSkip = await new Promise((resolve, reject) => { + this.hooks.execPre('bulkWrite', this, [ops, options], (err) => { + if (err != null) { + if (err instanceof Kareem.skipWrappedFunction) { + return resolve(err); + } + return reject(err); + } + resolve(); + }); + }); + + if (shouldSkip) { + return shouldSkip.args[0]; + } + const ordered = options.ordered == null ? true : options.ordered; + if (ops.length === 0) { + return getDefaultBulkwriteResult(); + } + const validations = ops.map(op => castBulkWrite(this, op, options)); - return new Promise((resolve, reject) => { - if (ordered) { + let res = null; + if (ordered) { + await new Promise((resolve, reject) => { each(validations, (fn, cb) => fn(cb), error => { if (error) { return reject(error); } - if (ops.length === 0) { - return resolve(getDefaultBulkwriteResult()); - } - - try { - this.$__collection.bulkWrite(ops, options, (error, res) => { - if (error) { - return reject(error); - } - - resolve(res); - }); - } catch (err) { - return reject(err); - } + resolve(); }); + }); - return; + try { + res = await this.$__collection.bulkWrite(ops, options); + } catch (error) { + await new Promise((resolve, reject) => { + const _opts = { error: error }; + this.hooks.execPost('bulkWrite', this, [null], _opts, (err) => { + if (err != null) { + return reject(err); + } + resolve(); + }); + }); } - + } else { let remaining = validations.length; let validOps = []; let validationErrors = []; const results = []; - if (remaining === 0) { - completeUnorderedValidation.call(this); - } else { + await new Promise((resolve) => { for (let i = 0; i < validations.length; ++i) { validations[i]((err) => { if (err == null) { @@ -3475,56 +3524,74 @@ Model.bulkWrite = async function bulkWrite(ops, options) { results[i] = err; } if (--remaining <= 0) { - completeUnorderedValidation.call(this); + resolve(); } }); } - } + }); validationErrors = validationErrors. sort((v1, v2) => v1.index - v2.index). map(v => v.error); - function completeUnorderedValidation() { - const validOpIndexes = validOps; - validOps = validOps.sort().map(index => ops[index]); + const validOpIndexes = validOps; + validOps = validOps.sort().map(index => ops[index]); - if (validOps.length === 0) { - return resolve(getDefaultBulkwriteResult()); - } + if (validOps.length === 0) { + return getDefaultBulkwriteResult(); + } - this.$__collection.bulkWrite(validOps, options, (error, res) => { - if (error) { - if (validationErrors.length > 0) { - error.mongoose = error.mongoose || {}; - error.mongoose.validationErrors = validationErrors; - } + let error; + [res, error] = await this.$__collection.bulkWrite(validOps, options). + then(res => ([res, null])). + catch(err => ([null, err])); - return reject(error); - } + if (error) { + if (validationErrors.length > 0) { + error.mongoose = error.mongoose || {}; + error.mongoose.validationErrors = validationErrors; + } - for (let i = 0; i < validOpIndexes.length; ++i) { - results[validOpIndexes[i]] = null; - } - if (validationErrors.length > 0) { - if (options.throwOnValidationError) { - return reject(new MongooseBulkWriteError( - validationErrors, - results, - res, - 'bulkWrite' - )); - } else { - res.mongoose = res.mongoose || {}; - res.mongoose.validationErrors = validationErrors; - res.mongoose.results = results; + await new Promise((resolve, reject) => { + const _opts = { error: error }; + this.hooks.execPost('bulkWrite', this, [null], _opts, (err) => { + if (err != null) { + return reject(err); } - } - - resolve(res); + resolve(); + }); }); } + + for (let i = 0; i < validOpIndexes.length; ++i) { + results[validOpIndexes[i]] = null; + } + if (validationErrors.length > 0) { + if (options.throwOnValidationError) { + throw new MongooseBulkWriteError( + validationErrors, + results, + res, + 'bulkWrite' + ); + } else { + res.mongoose = res.mongoose || {}; + res.mongoose.validationErrors = validationErrors; + res.mongoose.results = results; + } + } + } + + await new Promise((resolve, reject) => { + this.hooks.execPost('bulkWrite', this, [res], (err) => { + if (err != null) { + return reject(err); + } + resolve(); + }); }); + + return res; }; /** diff --git a/test/model.middleware.test.js b/test/model.middleware.test.js index 7747167bb4b..92ca5224dee 100644 --- a/test/model.middleware.test.js +++ b/test/model.middleware.test.js @@ -457,4 +457,137 @@ describe('model middleware', function() { assert.equal(preCalled, 1); assert.equal(postCalled, 1); }); + + describe('createCollection middleware', function() { + it('calls createCollection hooks', async function() { + const schema = new Schema({ name: String }, { autoCreate: true }); + + const pre = []; + const post = []; + schema.pre('createCollection', function() { + pre.push(this); + }); + schema.post('createCollection', function() { + post.push(this); + }); + + const Test = db.model('Test', schema); + await Test.init(); + assert.equal(pre.length, 1); + assert.equal(pre[0], Test); + assert.equal(post.length, 1); + assert.equal(post[0], Test); + }); + + it('allows skipping createCollection from hooks', async function() { + const schema = new Schema({ name: String }, { autoCreate: true }); + + schema.pre('createCollection', function(next) { + next(mongoose.skipMiddlewareFunction()); + }); + + const Test = db.model('CreateCollectionHookTest', schema); + await Test.init(); + const collections = await db.listCollections(); + assert.equal(collections.length, 0); + }); + }); + + describe('bulkWrite middleware', function() { + it('calls bulkWrite hooks', async function() { + const schema = new Schema({ name: String }); + + const pre = []; + const post = []; + schema.pre('bulkWrite', function(next, ops) { + pre.push(ops); + next(); + }); + schema.post('bulkWrite', function(res) { + post.push(res); + }); + + const Test = db.model('Test', schema); + await Test.bulkWrite([{ + updateOne: { + filter: { name: 'foo' }, + update: { $set: { name: 'bar' } } + } + }]); + assert.equal(pre.length, 1); + assert.deepStrictEqual(pre[0], [{ + updateOne: { + filter: { name: 'foo' }, + update: { $set: { name: 'bar' } } + } + }]); + assert.equal(post.length, 1); + assert.equal(post[0].constructor.name, 'BulkWriteResult'); + }); + + it('allows updating ops', async function() { + const schema = new Schema({ name: String, prop: String }); + + schema.pre('bulkWrite', function(next, ops) { + ops[0].updateOne.filter.name = 'baz'; + next(); + }); + + const Test = db.model('Test', schema); + const { _id } = await Test.create({ name: 'baz' }); + await Test.bulkWrite([{ + updateOne: { + filter: { name: 'foo' }, + update: { $set: { prop: 'test prop value' } } + } + }]); + const { prop } = await Test.findById(_id).orFail(); + assert.equal(prop, 'test prop value'); + }); + + it('supports error handlers', async function() { + const schema = new Schema({ name: String, prop: String }); + + const errors = []; + schema.post('bulkWrite', function(err, res, next) { + errors.push(err); + next(); + }); + + const Test = db.model('Test', schema); + const { _id } = await Test.create({ name: 'baz' }); + await assert.rejects( + Test.bulkWrite([{ + insertOne: { + document: { + _id + } + } + }]), + /duplicate key error/ + ); + assert.equal(errors.length, 1); + assert.equal(errors[0].name, 'MongoBulkWriteError'); + assert.ok(errors[0].message.includes('duplicate key error'), errors[0].message); + }); + + it('supports skipping wrapped function', async function() { + const schema = new Schema({ name: String, prop: String }); + + schema.pre('bulkWrite', function(next) { + next(mongoose.skipMiddlewareFunction('skipMiddlewareFunction test')); + }); + + const Test = db.model('Test', schema); + const { _id } = await Test.create({ name: 'baz' }); + const res = await Test.bulkWrite([{ + insertOne: { + document: { + _id + } + } + }]); + assert.strictEqual(res, 'skipMiddlewareFunction test'); + }); + }); }); diff --git a/test/types/middleware.test.ts b/test/types/middleware.test.ts index d06530e896f..31e210eb26d 100644 --- a/test/types/middleware.test.ts +++ b/test/types/middleware.test.ts @@ -1,5 +1,6 @@ import { Schema, model, Model, Document, SaveOptions, Query, Aggregate, HydratedDocument, PreSaveMiddlewareFunction, ModifyResult } from 'mongoose'; import { expectError, expectType, expectNotType, expectAssignable } from 'tsd'; +import { AnyBulkWriteOperation, CreateCollectionOptions } from 'mongodb'; const preMiddlewareFn: PreSaveMiddlewareFunction = function(next, opts) { this.$markValid('name'); @@ -90,6 +91,14 @@ schema.pre>('insertMany', function(next, docs: Array) { next(); }); +schema.pre>('bulkWrite', function(next, ops: Array>) { + next(); +}); + +schema.pre>('createCollection', function(next, opts?: CreateCollectionOptions) { + next(); +}); + schema.pre>('estimatedDocumentCount', function(next) {}); schema.post>('estimatedDocumentCount', function(count, next) { expectType(count); diff --git a/types/index.d.ts b/types/index.d.ts index 987d35c8117..6a5a926c654 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -373,8 +373,8 @@ declare module 'mongoose' { // method aggregate and insertMany with ErrorHandlingMiddlewareFunction post>(method: 'aggregate' | RegExp, fn: ErrorHandlingMiddlewareFunction>): this; post>(method: 'aggregate' | RegExp, options: SchemaPostOptions, fn: ErrorHandlingMiddlewareFunction>): this; - post(method: 'insertMany' | RegExp, fn: ErrorHandlingMiddlewareFunction): this; - post(method: 'insertMany' | RegExp, options: SchemaPostOptions, fn: ErrorHandlingMiddlewareFunction): this; + post(method: 'bulkWrite' | 'createCollection' | 'insertMany' | RegExp, fn: ErrorHandlingMiddlewareFunction): this; + post(method: 'bulkWrite' | 'createCollection' | 'insertMany' | RegExp, options: SchemaPostOptions, fn: ErrorHandlingMiddlewareFunction): this; /** Defines a pre hook for the model. */ // this = never since it never happens @@ -429,6 +429,44 @@ declare module 'mongoose' { options?: InsertManyOptions & { lean?: boolean } ) => void | Promise ): this; + /* method bulkWrite */ + pre( + method: 'bulkWrite' | RegExp, + fn: ( + this: T, + next: (err?: CallbackError) => void, + ops: Array & MongooseBulkWritePerWriteOptions>, + options?: mongodb.BulkWriteOptions & MongooseBulkWriteOptions + ) => void | Promise + ): this; + pre( + method: 'bulkWrite' | RegExp, + options: SchemaPreOptions, + fn: ( + this: T, + next: (err?: CallbackError) => void, + ops: Array & MongooseBulkWritePerWriteOptions>, + options?: mongodb.BulkWriteOptions & MongooseBulkWriteOptions + ) => void | Promise + ): this; + /* method createCollection */ + pre( + method: 'createCollection' | RegExp, + fn: ( + this: T, + next: (err?: CallbackError) => void, + options?: mongodb.CreateCollectionOptions & Pick + ) => void | Promise + ): this; + pre( + method: 'createCollection' | RegExp, + options: SchemaPreOptions, + fn: ( + this: T, + next: (err?: CallbackError) => void, + options?: mongodb.CreateCollectionOptions & Pick + ) => void | Promise + ): this; /** Object of currently defined query helpers on this schema. */ query: TQueryHelpers; From c164d63fef75867e8d3b0add164b26191e9795f5 Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Mon, 19 Feb 2024 16:29:43 -0500 Subject: [PATCH 23/24] Update lib/query.js Co-authored-by: hasezoey --- lib/query.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index 245c3d48d49..179b2924b5c 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2476,7 +2476,6 @@ Query.prototype._completeOne = function(doc, res, callback) { * Given a model and an array of docs, hydrates all the docs to be instances * of the model. Used to initialize docs returned from the db from `find()` * - * @param {Model} model * @param {Array} docs * @param {Object} fields the projection used, including `select` from schemas * @param {Object} userProvidedFields the user-specified projection From 4876d1e6b77be92ebe4ddb78315113a6482ee01c Mon Sep 17 00:00:00 2001 From: Valeri Karpov Date: Mon, 19 Feb 2024 16:29:48 -0500 Subject: [PATCH 24/24] Update lib/query.js Co-authored-by: hasezoey --- lib/query.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/query.js b/lib/query.js index 179b2924b5c..8731f003204 100644 --- a/lib/query.js +++ b/lib/query.js @@ -2482,7 +2482,6 @@ Query.prototype._completeOne = function(doc, res, callback) { * @param {Object} [opts] * @param {Array} [opts.populated] * @param {ClientSession} [opts.session] - * @param {Function} callback * @api private */