diff --git a/app/api/migrations/migrations/22-files-to-updatelogs/index.js b/app/api/migrations/migrations/22-files-to-updatelogs/index.js new file mode 100644 index 0000000000..f05c4967b3 --- /dev/null +++ b/app/api/migrations/migrations/22-files-to-updatelogs/index.js @@ -0,0 +1,41 @@ +/* eslint-disable no-await-in-loop, max-statements */ +export default { + delta: 22, + + name: 'files-to-updatelogs', + + description: 'update previously migrated files to updatelogs', + + async up(db) { + process.stdout.write(`${this.name}...\r\n`); + const cursor = db.collection('files').find({}); + + let index = 1; + + const [sync] = await db + .collection('syncs') + .find() + .toArray(); + + while (await cursor.hasNext()) { + const file = await cursor.next(); + const logExists = ( + await db + .collection('updatelogs') + .find({ mongoId: file._id }) + .toArray() + ).length; + + if (!logExists) { + await db.collection('updatelogs').insertOne({ + timestamp: sync ? sync.lastSync : 0, + namespace: 'files', + mongoId: file._id, + deleted: false, + }); + } + process.stdout.write(` -> processed: ${index} \r`); + index += 1; + } + }, +}; diff --git a/app/api/migrations/migrations/22-files-to-updatelogs/specs/22-files-to-updatelogs.spec.js b/app/api/migrations/migrations/22-files-to-updatelogs/specs/22-files-to-updatelogs.spec.js new file mode 100644 index 0000000000..c7992b3bbf --- /dev/null +++ b/app/api/migrations/migrations/22-files-to-updatelogs/specs/22-files-to-updatelogs.spec.js @@ -0,0 +1,78 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import fixtures, { file1, file2, file3 } from './fixtures.js'; + +const query = (collectionName, queryObject = {}, select = {}) => + testingDB.mongodb + .collection(collectionName) + .find(queryObject, select) + .toArray(); + +describe('migration files-to-updatelogs', () => { + beforeEach(async () => { + spyOn(process.stdout, 'write'); + await testingDB.clearAllAndLoad(fixtures); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(22); + }); + + it('should create entries on updatelogs for all files', async () => { + await migration.up(testingDB.mongodb); + const fileLogs = await query('updatelogs'); + + expect(fileLogs.length).toBe(3); + expect(fileLogs).toEqual([ + expect.objectContaining({ + timestamp: 50, + namespace: 'files', + mongoId: file3, + deleted: false, + }), + expect.objectContaining({ + timestamp: 0, + namespace: 'files', + mongoId: file1, + deleted: false, + }), + expect.objectContaining({ + timestamp: 0, + namespace: 'files', + mongoId: file2, + deleted: false, + }), + ]); + }); + + describe('when it has lastSync', () => { + it('should use lastSync as timestamp', async () => { + await testingDB.mongodb.collection('syncs').insertOne({ + lastSync: 20, + }); + + await migration.up(testingDB.mongodb); + const [file3log, file1log, file2log] = await query('updatelogs'); + + expect(file3log).toEqual( + expect.objectContaining({ + timestamp: 50, + }) + ); + expect(file1log).toEqual( + expect.objectContaining({ + timestamp: 20, + }) + ); + expect(file2log).toEqual( + expect.objectContaining({ + timestamp: 20, + }) + ); + }); + }); +}); diff --git a/app/api/migrations/migrations/22-files-to-updatelogs/specs/fixtures.js b/app/api/migrations/migrations/22-files-to-updatelogs/specs/fixtures.js new file mode 100644 index 0000000000..95e0b11d73 --- /dev/null +++ b/app/api/migrations/migrations/22-files-to-updatelogs/specs/fixtures.js @@ -0,0 +1,28 @@ +import db from 'api/utils/testing_db'; + +const [file1, file2, file3] = [db.id(), db.id(), db.id()]; + +export default { + files: [ + { + _id: file1, + }, + { + _id: file2, + }, + { + _id: file3, + }, + ], + updatelogs: [ + { + _id: db.id(), + timestamp: 50, + namespace: 'files', + mongoId: file3, + deleted: false, + }, + ], +}; + +export { file1, file2, file3 }; diff --git a/app/api/migrations/migrations/23-fix_udaptelogs/index.js b/app/api/migrations/migrations/23-fix_udaptelogs/index.js new file mode 100644 index 0000000000..f9c6253ae8 --- /dev/null +++ b/app/api/migrations/migrations/23-fix_udaptelogs/index.js @@ -0,0 +1,12 @@ +export default { + delta: 23, + + name: 'fix_udaptelogs', + + description: 'delete update logs without mongoId', + + async up(db) { + process.stdout.write(`${this.name}...\r\n`); + await db.collection('updatelogs').removeMany({ mongoId: { $exists: false } }); + }, +}; diff --git a/app/api/migrations/migrations/23-fix_udaptelogs/specs/23-fix_udaptelogs.spec.js b/app/api/migrations/migrations/23-fix_udaptelogs/specs/23-fix_udaptelogs.spec.js new file mode 100644 index 0000000000..910321076f --- /dev/null +++ b/app/api/migrations/migrations/23-fix_udaptelogs/specs/23-fix_udaptelogs.spec.js @@ -0,0 +1,37 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import fixtures from './fixtures.js'; + +describe('migration fix_udaptelogs', () => { + beforeEach(async () => { + spyOn(process.stdout, 'write'); + await testingDB.clearAllAndLoad(fixtures); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(23); + }); + + it('should remove all updatelogs without mongoId', async () => { + await migration.up(testingDB.mongodb); + const updatelogs = await testingDB.mongodb + .collection('updatelogs') + .find() + .toArray(); + + expect(updatelogs).toEqual([ + expect.objectContaining({ + mongoId: expect.anything(), + namespace: 'entities', + }), + expect.objectContaining({ + mongoId: expect.anything(), + namespace: 'entities', + }), + ]); + }); +}); diff --git a/app/api/migrations/migrations/23-fix_udaptelogs/specs/fixtures.js b/app/api/migrations/migrations/23-fix_udaptelogs/specs/fixtures.js new file mode 100644 index 0000000000..bab08bcdb4 --- /dev/null +++ b/app/api/migrations/migrations/23-fix_udaptelogs/specs/fixtures.js @@ -0,0 +1,20 @@ +import db from 'api/utils/testing_db'; + +export default { + updatelogs: [ + { + mongoId: db.id(), + namespace: 'entities', + }, + { + mongoId: db.id(), + namespace: 'entities', + }, + { + namespace: 'entities', + }, + { + namespace: 'entities', + }, + ], +}; diff --git a/app/api/migrations/templates/migration.spec.txt b/app/api/migrations/templates/migration.spec.txt index 5b3470ee46..6335eecb28 100644 --- a/app/api/migrations/templates/migration.spec.txt +++ b/app/api/migrations/templates/migration.spec.txt @@ -1,23 +1,22 @@ -import { catchErrors } from 'api/utils/jasmineHelpers'; import testingDB from 'api/utils/testing_db'; import migration from '../index.js'; import fixtures from './fixtures.js'; describe('migration {{ name }}', () => { - beforeEach((done) => { + beforeEach(async () => { spyOn(process.stdout, 'write'); - testingDB.clearAllAndLoad(fixtures).then(done).catch(catchErrors(done)); + await testingDB.clearAllAndLoad(fixtures); }); - afterAll((done) => { - testingDB.disconnect().then(done); + afterAll(async () => { + await testingDB.disconnect(); }); it('should have a delta number', () => { expect(migration.delta).toBe({{nextMigrationDelta}}); }); - it('should fail', (done) => { - migration.up().catch(catchErrors(done)); + it('should fail', async () => { + await migration.up(); }); }); diff --git a/app/api/migrations/templates/migration.txt b/app/api/migrations/templates/migration.txt index 7563dfcef9..973dad34a0 100644 --- a/app/api/migrations/templates/migration.txt +++ b/app/api/migrations/templates/migration.txt @@ -5,8 +5,8 @@ export default { description: '{{ description }}', - up() { + async up() { process.stdout.write(`${this.name}...\r\n`); - return Promise.reject(); - } + return Promise.reject(new Error('error! change this, recently created migration')); + }, }; diff --git a/app/api/odm/model.ts b/app/api/odm/model.ts index 47e86c0bff..46dcb91c5c 100644 --- a/app/api/odm/model.ts +++ b/app/api/odm/model.ts @@ -1,6 +1,5 @@ -/** @format */ - import mongoose from 'mongoose'; +import errorLog from 'api/log/errorLog'; import { model as updatelogsModel } from 'api/updatelogs'; @@ -32,8 +31,16 @@ class UpdateLogHelper { } upsertLogOne(doc: mongoose.Document, next: (err?: mongoose.NativeError) => void) { + // temporary prevent creating updateLogs without mongoid, log error + if (!doc._id) { + errorLog.error( + `ERROR, Update Log received a doc without _id:\n ${JSON.stringify(doc, null, ' ')}` + ); + return next(); + } + // const logData = { namespace: this.collectionName, mongoId: doc._id }; - asyncPost(async () => { + return asyncPost(async () => { await updatelogsModel.findOneAndUpdate( logData, { ...logData, timestamp: Date.now(), deleted: false }, diff --git a/app/api/updatelogs/updatelogsModel.ts b/app/api/updatelogs/updatelogsModel.ts index 07aced5455..7b2df18ac2 100644 --- a/app/api/updatelogs/updatelogsModel.ts +++ b/app/api/updatelogs/updatelogsModel.ts @@ -1,5 +1,3 @@ -/** @format */ - import mongoose from 'mongoose'; const updateLogSchema = new mongoose.Schema({ diff --git a/nightmare/paths/metadataprops.spec.js b/nightmare/paths/metadataprops.spec.js index 1771452e56..8478a97f44 100644 --- a/nightmare/paths/metadataprops.spec.js +++ b/nightmare/paths/metadataprops.spec.js @@ -128,12 +128,14 @@ describe('metadata properties', () => { .waitToClick(selectors.settingsView.addNewTemplate) .write(selectors.settingsView.documentTemplateNameForm, 'All props'); - for (let propIndex = 1; propIndex <= 15; propIndex += 1) { + //intentionaly leaving the geolocation field out of the test. + for (let propIndex = 1; propIndex <= 14; propIndex += 1) { await nightmare.waitToClick(localSelectors.propertiesButtons(propIndex)); } await nightmare .waitToClick(localSelectors.editRelationshipProperty) + .wait(localSelectors.relationshipPropertyType) .selectByLabel(localSelectors.relationshipPropertyType, 'Perpetrator') .waitToClick(selectors.settingsView.saveTemplateButton) .waitToClick('.alert.alert-success'); @@ -167,8 +169,6 @@ describe('metadata properties', () => { .type(localSelectors.form.linkUrl, 'https://www.huridocs.org/') .type(localSelectors.form.image, '/public/logo.svg') .type(localSelectors.form.media, 'test') - .insert(localSelectors.form.geolocationLat, '46') - .insert(localSelectors.form.geolocationLon, '6') .click(localSelectors.form.save) .waitToClick('.alert.alert-success'); }, 60000); @@ -221,8 +221,7 @@ describe('metadata properties', () => { return nightmare .wait(localSelectors.properties.image) .wait(localSelectors.properties.preview) - .wait(localSelectors.properties.media) - .wait(localSelectors.properties.geolocation); + .wait(localSelectors.properties.media); }); }, 60000); @@ -247,8 +246,6 @@ describe('metadata properties', () => { .clearInput(localSelectors.form.linkUrl) .clearInput(localSelectors.form.image) .clearInput(localSelectors.form.media) - .clearInput(localSelectors.form.geolocationLat) - .clearInput(localSelectors.form.geolocationLon) .click(localSelectors.form.save) .waitToClick('.alert.alert-success'); }, 60000);