From d77526e8d49c717780f09327fd7511f12185f696 Mon Sep 17 00:00:00 2001 From: Evgeniia Rassokhina Date: Fri, 9 Jul 2021 23:42:01 +0300 Subject: [PATCH 01/15] The synchronization to work in transactions between Arbimon and RFCx is updated --- CHANGELOG.md | 6 ++ app/model/projects.js | 142 +++++++++++++++++++++---- app/model/sites.js | 134 +++++++++++++++++++---- app/routes/data-api/orders.js | 51 +-------- app/routes/data-api/project/index.js | 14 +-- app/routes/data-api/project/sites.js | 70 +++--------- assets/app/a2services/sites-service.js | 12 ++- assets/app/app/audiodata/sites.js | 2 +- 8 files changed, 272 insertions(+), 159 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c8ae34eb..4463a3227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Arbimon Release Notes +## v3.0.32 - June XX, 2021 + +Resolved issues: + +- The synchronization to work in transactions between Arbimon and RFCx is updated + ## v3.0.31 - June 01, 2021 New features: diff --git a/app/model/projects.js b/app/model/projects.js index 269d259d8..69393f19b 100644 --- a/app/model/projects.js +++ b/app/model/projects.js @@ -252,7 +252,7 @@ var Projects = { * @param {Boolean} project.is_private * @param {Function} callback(err, projectId) */ - create: function(project, owner_id, callback) { + create: async function(project, owner_id, db, callback) { var schema = joi.object().keys({ name: joi.string(), url: joi.string(), @@ -299,12 +299,50 @@ var Projects = { 'SET current_plan = ? \n'+ 'WHERE project_id = ?'; + var projectId; + if (db) { + + async.waterfall([ + function insertProject(cb) { + db.query(q, project, cb); + }, + function insertOwner(result, fields, cb) { + projectId = result.insertId; + + var values = { + user_id: owner_id, + project_id: projectId, + role_id: 4 // owner role id + }; + + db.query(q2, values, cb); + }, + function insertPlan(result, fields, cb) { + plan.project_id = projectId; + plan.created_on = new Date(); + + db.query(createPlan, plan, cb); + }, + function updateCurrentPlan(result, fields, cb) { + db.query(updatePlan, [result.insertId, projectId], cb); + }, + function commit(result, fields, cb) { + db.commit(cb); + } + ], + function(err) { + if(err) { + callback(err); + return; + } + + callback(null, projectId); + }); + } else { dbpool.getConnection(function(err, db) { db.beginTransaction(function(err){ if(err) return callback(err); - var projectId; - async.waterfall([ function insertProject(cb) { db.query(q, project, cb); @@ -346,6 +384,7 @@ var Projects = { }); }); }); + } }, /** @@ -365,7 +404,7 @@ var Projects = { * * @return {Promise} resolved after the update. */ - update: function(project, callback) { + update: function(project, db, callback) { var schema = { project_id: joi.number().required(), @@ -400,13 +439,13 @@ var Projects = { delete projectInfo.plan; return q.all([ - dbpool.query( + db.query( 'UPDATE projects\n'+ 'SET ?\n'+ 'WHERE project_id = ?', [ projectInfo, projectId ]), - projectInfoPlan && dbpool.query( + projectInfoPlan && db.query( "UPDATE project_plans\n"+ "SET ?\n" + "WHERE project_id=?", [ @@ -942,6 +981,31 @@ var Projects = { ).get(0); }, + createProjectInArbimonAndCoreAPI: async function(project, userId, token) { + let connection; + return dbpool.getConnection() + .then(async (con) => { + connection = con; + await connection.beginTransaction(); + let newProject = await this.createProject(project, userId, connection); + project.project_id = newProject.project_id; + if (rfcxConfig.coreAPIEnabled) { + let externalProjectId = await this.createInCoreAPI(project, token); + await this.setExternalId(project.project_id, externalProjectId, connection); + } + await connection.commit() + await connection.release() + }) + .catch(async (err) => { + console.log('Failed to create project', err); + if (connection) { + await connection.rollback(); + await connection.release(); + throw new APIError('Failed to create project'); + } + }) + }, + createInCoreAPI: async function(project, idToken) { const body = { name: project.name, @@ -972,6 +1036,30 @@ var Projects = { }) }, + updateProjectInArbimonAndCoreAPI: async function(data, token) { + let connection; + return dbpool.getConnection() + .then(async (con) => { + connection = con; + await connection.beginTransaction(); + await this.update(data, connection); + if (rfcxConfig.coreAPIEnabled) { + await this.updateInCoreAPI(data, token); + } + await connection.commit() + await connection.release() + }) + .catch(async (err) => { + console.log('Failed to update project', err); + if (connection) { + await connection.rollback(); + await connection.release(); + throw new APIError('Failed to update project'); + } + }) + }, + + updateInCoreAPI: async function(data, idToken) { let body = {} data.name !== undefined && (body.name = data.name) @@ -986,8 +1074,12 @@ var Projects = { source: 'arbimon' }, body: JSON.stringify(body) - } - return rp(options) + } + return rp(options).then((response) => { + if (response.statusCode === 403) { + throw new Error('Forbidden error.') + } + }) }, deleteInCoreAPI: async function(project_id, idToken) { @@ -1001,8 +1093,12 @@ var Projects = { source: 'arbimon' }, body: JSON.stringify(body) - } - return rp(options) + } + return rp(options).then((response) => { + if (response.statusCode === 403) { + throw new Error('Forbidden error.') + } + }) }, findInCoreAPI: async function (guid) { @@ -1025,8 +1121,8 @@ var Projects = { }) }, - setExternalId: function (projectId, externalId) { - return dbpool.query(`UPDATE projects SET external_id = "${externalId}" WHERE project_id = ${projectId}`, []) + setExternalId: function (projectId, externalId, connection) { + return (connection? connection.query : dbpool.query)(`UPDATE projects SET external_id = "${externalId}" WHERE project_id = ${projectId}`, []) }, /** @@ -1056,7 +1152,7 @@ var Projects = { * @param {boolean} data.is_private * @param {integer} userId */ - createProject: async function (data, userId) { + createProject: async function (data, userId, connection) { const projectData = { plan: this.plans.free, project_type_id: 1, @@ -1066,7 +1162,8 @@ var Projects = { stripUnknown: true, presence: 'required', }); - const id = await q.ninvoke(this, "create", projectData, userId) + // const id = await this.create(projectData, userId, connection); + const id = await q.ninvoke(this, "create", projectData, userId, connection); return this.find({ id }).get(0) }, @@ -1074,20 +1171,21 @@ var Projects = { let db; return dbpool.getConnection() .then(async (connection) => { - db = connection - await db.beginTransaction() - await this.deleteInArbimobDb(options.project_id, connection) + db = connection; + await db.beginTransaction(); + await this.deleteInArbimobDb(options.project_id, connection); if (rfcxConfig.coreAPIEnabled) { await this.deleteInCoreAPI(options.external_id, options.idToken) - } - await db.commit() - await db.release() + }; + await db.commit(); + await db.release(); }) .catch(async (err) => { console.log('err', err) if (db) { - await db.rollback() - await db.release() + await db.rollback(); + await db.release(); + throw new APIError('Failed to delete project', 422); } }) }, diff --git a/app/model/sites.js b/app/model/sites.js index 56331070c..39c99763f 100644 --- a/app/model/sites.js +++ b/app/model/sites.js @@ -13,7 +13,7 @@ var s3; var dbpool = require('../utils/dbpool'); var queryHandler = dbpool.queryHandler; const moment = require('moment'); - +let APIError = require('../utils/apierror'); var site_log_processor = require('../utils/site_log_processor'); const projects = require('./projects') @@ -64,7 +64,7 @@ var Sites = { return find(site_id) }, - insert: function(site, callback) { + insert: function(site, db, callback) { var values = []; var schema = { @@ -109,15 +109,15 @@ var Sites = { 'SET %s'; q = util.format(q, values.join(", ")); - queryHandler(q, callback); + db ? db.query(q, callback) : queryHandler(q, callback); }, - insertAsync: function(site) { + insertAsync: function(site, connection) { let insert = util.promisify(this.insert) - return insert(site) + return insert(site, connection) }, - update: async function(site, callback) { + update: async function(site, connection, callback) { var values = []; if(site.id) @@ -169,12 +169,12 @@ var Sites = { q = util.format(q, values.join(", "), site.site_id); - queryHandler(q, callback); + connection ? connection.query(q, callback) : queryHandler(q, callback); }, - updateAsync: function (site) { + updateAsync: function (site, connection) { let update = util.promisify(this.update) - return update(site) + return update(site, connection) }, exists: function(site_name, project_id, callback) { @@ -195,7 +195,7 @@ var Sites = { }); }, - removeFromProject: function(site_id, project_id, callback) { + removeFromProject: function(site_id, project_id, connection, callback) { if(!site_id || !project_id) return callback(new Error("required field missing")); @@ -220,7 +220,7 @@ var Sites = { 'WHERE site_id = %s'; q = util.format(q, dbpool.escape(site_id)); - queryHandler(q, callback); + connection? connection.query(q, callback): queryHandler(q, callback); } }); } @@ -230,14 +230,14 @@ var Sites = { 'AND project_id = %s'; q = util.format(q, dbpool.escape(site_id), dbpool.escape(project_id)); - queryHandler(q, callback); + connection? connection.query(q, callback): queryHandler(q, callback); } }); }, - removeFromProjectAsync: function (siteId, projectId) { + removeFromProjectAsync: function (siteId, projectId, connection) { let remove = util.promisify(this.removeFromProject) - return remove(siteId, projectId) + return remove(siteId, projectId, connection) }, listPublished: function(callback) { @@ -654,6 +654,40 @@ var Sites = { callback); }, + createSiteInArbimonAndCoreAPI: async function(site, projectExternalId, token) { + var connection; + return dbpool.getConnection() + .then(async (con) => { + connection = con; + await connection.beginTransaction(); + let result = await this.insertAsync(site, connection); + if (rfcxConfig.coreAPIEnabled) { + const coreSite = { + site_id: result.insertId, + name: site.name, + lat: site.lat, + lon: site.lon, + alt: site.alt + } + if (projectExternalId) { + coreSite.project_id = projectExternalId; + } + let siteExternalId = await this.createInCoreAPI(coreSite, token); + await this.setExternalId(result.insertId, siteExternalId, connection); + } + await connection.commit(); + await connection.release(); + }) + .catch(async (err) => { + console.log('Failed to create site', err); + if (connection) { + await connection.rollback(); + await connection.release(); + throw new APIError('Failed to create site'); + } + }) + }, + createInCoreAPI: async function(site, idToken) { const body = { name: site.name, @@ -686,6 +720,37 @@ var Sites = { }) }, + updateSite: async function(site, user, idToken) { + let db; + return dbpool.getConnection() + .then(async (connection) => { + db = connection; + await db.beginTransaction(); + await this.updateAsync(site, connection); + let personalProject = await model.projects.findOrCreatePersonalProject(user); + if (rfcxConfig.coreAPIEnabled) { + await this.updateInCoreAPI({ + site_id: site.site_id, + name: site.name, + lat: site.lat, + lon: site.lon, + alt: site.alt, + ...personalProject && personalProject.project_id === site.project_id ? {} : { project_id: site.project_id } + }, idToken) + }; + await db.commit(); + await db.release(); + }) + .catch(async (err) => { + console.log('Failed to update site', err); + if (db) { + await db.rollback(); + await db.release(); + throw new APIError('Failed to update site'); + } + }) + }, + updateInCoreAPI: async function(data, idToken) { let body = {} data.name !== undefined && (body.name = data.name) @@ -702,8 +767,35 @@ var Sites = { source: 'arbimon' }, body: JSON.stringify(body) - } - return rp(options) + } + return rp(options).then((response) => { + if (response.statusCode === 403) { + throw new Error('Forbidden error.') + } + }) + }, + + removeSite: async function(site_id, project_id, idToken) { + let db; + return dbpool.getConnection() + .then(async (connection) => { + db = connection; + await db.beginTransaction(); + await this.removeFromProjectAsync(site_id, project_id, connection); + if (rfcxConfig.coreAPIEnabled) { + await this.deleteInCoreAPI(site_id, idToken) + }; + await db.commit(); + await db.release(); + }) + .catch(async (err) => { + console.log('Failed to delete site', err); + if (db) { + await db.rollback(); + await db.release(); + throw new APIError('Failed to delete site'); + } + }) }, deleteInCoreAPI: async function(site_id, idToken) { @@ -716,7 +808,11 @@ var Sites = { source: 'arbimon' } } - return rp(options) + return rp(options).then((response) => { + if (response.statusCode === 403) { + throw new Error('Forbidden error.') + } + }) }, findInCoreAPI: async function (guid) { @@ -734,8 +830,8 @@ var Sites = { return rp(options).then(({ body }) => body) }, - setExternalId: function (siteId, externalId) { - return dbpool.query(`UPDATE sites SET external_id = "${externalId}" WHERE site_id = ${siteId}`, []) + setExternalId: function (siteId, externalId, connection) { + return (connection? connection.query : dbpool.query)(`UPDATE sites SET external_id = "${externalId}" WHERE site_id = ${siteId}`, []) }, /** diff --git a/app/routes/data-api/orders.js b/app/routes/data-api/orders.js index 06d411970..09e66afaf 100644 --- a/app/routes/data-api/orders.js +++ b/app/routes/data-api/orders.js @@ -21,22 +21,6 @@ var ordersUtils = require('../../utils/orders.js'); var countries = require('../../utils/countries.js'); var shippingCalculator = require('../../utils/shipping-calculator.js'); -/** - creates a new project an create news about project creation -*/ -var createProject = function(project, userId) { - return q.ninvoke(model.projects, "create", project, userId).then(function(projectId) { - model.projects.insertNews({ - news_type_id: 1, // project created - user_id: userId, - project_id: projectId, - data: JSON.stringify({}) - }); - - return projectId; - }); -}; - var findLinkObject = function(links, linkRelation) { var approvalLink = links.filter(function(link) { if(link.rel == linkRelation) @@ -208,22 +192,11 @@ router.post('/create-project', function(req, res, next) { tier : 'paid', }; - return createProject(project, req.session.user.id) + return model.projects.createProjectInArbimonAndCoreAPI(project, req.session.user.id, req.session.idToken) .then(async function (projectId) { await model.ActivationCodes.consumeCode(coupon, req.session.user.id); - return projectId - }).then(function (projectId) { - if (req.session.user && req.session.user.rfcx_id) { - project.project_id = projectId - if (rfcxConfig.coreAPIEnabled) { - return model.projects.createInCoreAPI(project, req.session.idToken) - .then((externalProjectId) => { - return model.projects.setExternalId(project.project_id, externalProjectId) - }) - .catch((err) => console.error(`Failed to create project in Core: ${err.message}`)) - } - } - }).then(function () { + }) + .then(function () { res.json({ message: util.format("Project '%s' successfully created!", project.name) }); @@ -243,19 +216,8 @@ router.post('/create-project', function(req, res, next) { } }).then(function(){ project.plan = plan; - return createProject(project, req.session.user.id); + return model.projects.createProjectInArbimonAndCoreAPI(project, req.session.user.id, req.session.idToken) }).then(function (projectId) { - if (req.session.user && req.session.user.rfcx_id) { - project.project_id = projectId - if (rfcxConfig.coreAPIEnabled) { - return model.projects.createInCoreAPI(project, req.session.idToken) - .then((externalProjectId) => { - return model.projects.setExternalId(project.project_id, externalProjectId) - }) - .catch((err) => console.error(`Failed to create project in Core: ${err.message}`)) - } - } - }).then(function () { res.json({ message: util.format("Project '%s' successfully created!", project.name) }); @@ -534,10 +496,7 @@ router.post('/process/:orderId', function(req, res, next) { // create project and plan for it if(req.order.action == 'create-project') { - createProject( - req.order.data.project, - req.session.user.id - ).nodeify(cb); + model.projects.createProjectInArbimonAndCoreAPI(req.order.data.project, req.session.user.id, req.session.idToken).nodeify(cb); } // update project plan diff --git a/app/routes/data-api/project/index.js b/app/routes/data-api/project/index.js index cdc6bcf51..87529510d 100644 --- a/app/routes/data-api/project/index.js +++ b/app/routes/data-api/project/index.js @@ -154,17 +154,9 @@ router.post('/:projectUrl/info/update', function(req, res, next) { } }, function(urlChanged, callback) { - model.projects.update(newProjectInfo, function(err, result){ - if(err) return next(err); - - var url = urlChanged ? newProjectInfo.url : undefined; - - debug("update project:", result); - if (rfcxConfig.coreAPIEnabled) { - model.projects.updateInCoreAPI(newProjectInfo, req.session.idToken) - } - res.json({ success: true , url: url }); - }); + model.projects.updateProjectInArbimonAndCoreAPI(newProjectInfo, req.session.idToken); + var url = urlChanged ? newProjectInfo.url : undefined; + res.json({ success: true , url: url }); } ]); }); diff --git a/app/routes/data-api/project/sites.js b/app/routes/data-api/project/sites.js index 52df9173b..aa1b326d0 100644 --- a/app/routes/data-api/project/sites.js +++ b/app/routes/data-api/project/sites.js @@ -29,7 +29,7 @@ router.post('/create', function(req, res, next) { return res.json({ error: "you dont have permission to 'manage project sites'" }); } - model.sites.exists(site.name, project.project_id, function(err, exists) { + model.sites.exists(site.name, project.project_id, async function(err, exists) { if(err) return next(err); if(exists) @@ -37,27 +37,14 @@ router.post('/create', function(req, res, next) { site.project_id = project.project_id; - model.sites.insert(site, async function(err, result) { - if(err) return next(err); - - if (rfcxConfig.coreAPIEnabled) { - const coreSite = { - site_id: result.insertId, - name: site.name, - lat: site.lat, - lon: site.lon, - alt: site.alt - } - if (project.external_id) { - coreSite.project_id = project.external_id - } - await model.sites.createInCoreAPI(coreSite, req.session.idToken) - .then((externalSiteId) => model.sites.setExternalId(result.insertId, externalSiteId)) - .catch((err) => console.error(`Failed to create site in Core: ${err.message}`)) - } - + try { + await model.sites.createSiteInArbimonAndCoreAPI(site, project.external_id, req.session.idToken); res.json({ message: "New site created" }); - }); + } + catch(e) { + return next(err); + } + res.json({ message: "New site created" }); }); }); @@ -90,25 +77,9 @@ router.post('/update', function(req, res, next) { site.project_id = site.project? site.project.project_id : project.project_id; - model.sites.update(site, async function(err, rows) { - if(err) return next(err); - - if (rfcxConfig.coreAPIEnabled) { - await model.projects.findOrCreatePersonalProject({ ...req.session.user, user_id: req.session.user.id }) - .then((personalProject) => { - return model.sites.updateInCoreAPI({ - site_id: site.site_id, - name: site.name, - lat: site.lat, - lon: site.lon, - alt: site.alt, - ...personalProject && personalProject.project_id === site.project_id ? {} : { project_id: site.project_id } - }, req.session.idToken) - }) - } - - res.json({ message: "site updated" }); - }); + model.sites.updateSite(site, { ...req.session.user, user_id: req.session.user.id }, req.session.idToken).then(function() { + res.json({ message: 'Site updated' }); + }).catch(next); }); router.post('/delete', function(req, res, next) { @@ -120,22 +91,9 @@ router.post('/delete', function(req, res, next) { return res.json({ error: "you dont have permission to 'manage project sites'" }); } - model.sites.removeFromProject(site.id, project.project_id, function(err, rows) { - if(err) return next(err); - - model.projects.insertNews({ - news_type_id: 4, // site deleted - user_id: req.session.user.id, - project_id: project.project_id, - data: JSON.stringify({ sites: site.name }) - }); - - if (rfcxConfig.coreAPIEnabled) { - model.sites.deleteInCoreAPI(site.id, req.session.idToken) - } - - res.json(rows); - }); + model.sites.removeSite(site.id, project.project_id, req.session.idToken).then(function() { + res.json({ message: 'Site removed' }); + }).catch(next); }); router.param('siteid', function(req, res, next, siteid){ diff --git a/assets/app/a2services/sites-service.js b/assets/app/a2services/sites-service.js index e98ca4697..64a2cbe18 100644 --- a/assets/app/a2services/sites-service.js +++ b/assets/app/a2services/sites-service.js @@ -1,7 +1,8 @@ angular.module('a2.srv.sites', [ 'a2.srv.project', + 'humane' ]) -.factory('a2Sites',function($http, $q, Project){ +.factory('a2Sites',function($http, $q, Project, notify){ return { listPublished: function(callback) { $http.get('/api/sites/published') @@ -21,21 +22,24 @@ angular.module('a2.srv.sites', [ $http.post('/api/project/'+ Project.getUrl() +'/sites/update', { site: site, }) - .success(callback); + .success(callback) + .error(notify.serverError); }, create: function(site, callback) { $http.post('/api/project/'+ Project.getUrl() +'/sites/create', { site: site, }) - .success(callback); + .success(callback) + .error(notify.serverError); }, delete: function(site, callback) { $http.post('/api/project/'+ Project.getUrl() +'/sites/delete', { site: site }) - .success(callback); + .success(callback) + .error(notify.serverError); }, // Uses Promises :-) diff --git a/assets/app/app/audiodata/sites.js b/assets/app/app/audiodata/sites.js index 1a64e725f..846867453 100644 --- a/assets/app/app/audiodata/sites.js +++ b/assets/app/app/audiodata/sites.js @@ -296,7 +296,7 @@ angular.module('a2.audiodata.sites', [ Project.getSites({ count: true, logs: true }, function(sites) { $scope.sortByLastUpdated(sites); }); - notify.log("site removed"); + notify.log('Site removed'); }); }); }; From 7208bd846cef5438c5457089b8babc91874f9235 Mon Sep 17 00:00:00 2001 From: Evgeniia Rassokhina Date: Tue, 13 Jul 2021 14:14:48 +0300 Subject: [PATCH 02/15] CE-323 When user creates/edits/deletes a project role for another user in Arbimon it's synced with Core API in transactions --- CHANGELOG.md | 1 + app/model/projects.js | 66 +++++++++++++++++++++++++--- app/routes/data-api/project/index.js | 37 +++------------- 3 files changed, 67 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c0fbe7e8..aafb3e652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Arbimon Release Notes - CE-1093 The color of Species boxes is changed on the spectrogram +- CE-323 When user creates/edits/deletes a project role for another user in Arbimon it's synced with Core API in transactions ## v3.0.31 - June 01, 2021 diff --git a/app/model/projects.js b/app/model/projects.js index 269d259d8..7195aa0a7 100644 --- a/app/model/projects.js +++ b/app/model/projects.js @@ -642,7 +642,7 @@ var Projects = { return getUsers(project_id) }, - addUser: function(userProjectRole, callback) { + addUser: function(userProjectRole, connection, callback) { var schema = { user_id: joi.number().required(), project_id: joi.number().required(), @@ -670,14 +670,19 @@ var Projects = { var q = 'INSERT INTO user_project_role \n'+ 'SET user_id = %s, role_id = %s, project_id = %s'; q = util.format(q, user_id, role_id, project_id); - queryHandler(q, callback); + connection ? connection.query(q, callback) : queryHandler(q, callback); } }); }); }, - changeUserRole: function(userProjectRole, callback) { + addUserAsync: function(userProjectRole, connection) { + let addUser = util.promisify(this.addUser) + return addUser(userProjectRole, connection) + }, + + changeUserRole: function(userProjectRole, connection, callback) { var schema = { user_id: joi.number().required(), project_id: joi.number().required(), @@ -697,10 +702,54 @@ var Projects = { "AND project_id = %s"; q = util.format(q, role_id, user_id, project_id); - queryHandler(q, callback); + connection ? connection.query(q, callback) : queryHandler(q, callback); }); }, + changeUserRoleAsync: function(userProjectRole, connection) { + let changeUserRole = util.promisify(this.changeUserRole); + return changeUserRole(userProjectRole, connection); + }, + + updateUserRoleInArbimonAndCoreAPI: async function(options, token, action) { + let connection; + return dbpool.getConnection() + .then(async (con) => { + connection = con; + await connection.beginTransaction(); + switch (action) { + case 'add': + await this.addUserAsync(options.userRole, connection); + if (rfcxConfig.coreAPIEnabled) { + await this.updateUserRoleInCoreAPI(options.userRole, token); + } + break; + case 'change': + await this.changeUserRoleAsync(options.userRole, connection); + if (rfcxConfig.coreAPIEnabled) { + await this.updateUserRoleInCoreAPI(options.userRole, token); + } + break; + case 'remove': + await this.removeUserRoleAsync(options.user_id, options.project_id, connection); + if (rfcxConfig.coreAPIEnabled) { + await this.removeUserRoleInCoreAPI(options.user_id, options.project_id, token); + } + break; + } + await connection.commit(); + await connection.release(); + }) + .catch(async (err) => { + console.log('err', err); + if (connection) { + await connection.rollback(); + await connection.release(); + } + throw new APIError('Failed to update user project role'); + }) + }, + updateUserRoleInCoreAPI: async function(userProjectRole, idToken) { const project = await this.findById(userProjectRole.project_id) if (!project.external_id) { @@ -889,7 +938,7 @@ var Projects = { queryHandler(q, callback); }, - removeUser: function(user_id, project_id, callback) { + removeUser: function(user_id, project_id, connection, callback) { if(typeof project_id !== 'number') return callback(new Error("invalid type for 'project_id'")); @@ -900,7 +949,12 @@ var Projects = { "WHERE user_id = %s AND project_id = %s"; q = util.format(q, dbpool.escape(user_id), dbpool.escape(project_id)); - queryHandler(q, callback); + connection ? connection.query(q, callback) : queryHandler(q, callback); + }, + + removeUserRoleAsync: function(user_id, project_id, connection) { + let removeUser = util.promisify(this.removeUser) + return removeUser(user_id, project_id, connection) }, availableRoles: function(callback) { diff --git a/app/routes/data-api/project/index.js b/app/routes/data-api/project/index.js index cdc6bcf51..c75ef86dd 100644 --- a/app/routes/data-api/project/index.js +++ b/app/routes/data-api/project/index.js @@ -312,21 +312,9 @@ router.post('/:projectUrl/user/add', async function(req, res, next) { user_id: req.body.user_id, role_id: 2 // default to normal user } - model.projects.addUser(userRole, - async function(err, result){ - if (err) { - if (err.status === 404) { - return res.json({ error: err.message}); - } - return next(err); - } - - debug("add user:", result); - if (rfcxConfig.coreAPIEnabled) { - await model.projects.updateUserRoleInCoreAPI(userRole, req.session.idToken) - } + model.projects.updateUserRoleInArbimonAndCoreAPI({userRole: userRole}, req.session.idToken, 'add').then(function() { res.json({ success: true }); - }); + }).catch(next); }); router.post('/:projectUrl/user/role', async function(req, res, next) { @@ -345,16 +333,9 @@ router.post('/:projectUrl/user/role', async function(req, res, next) { user_id: req.body.user_id, role_id: req.body.role_id } - model.projects.changeUserRole(userRole, - async function(err, result){ - if(err) return next(err); - - debug("change user role:", result); - if (rfcxConfig.coreAPIEnabled) { - await model.projects.updateUserRoleInCoreAPI(userRole, req.session.idToken) - } + model.projects.updateUserRoleInArbimonAndCoreAPI({userRole: userRole}, req.session.idToken, 'change').then(function() { res.json({ success: true }); - }); + }).catch(next); }); router.post('/:projectUrl/user/del', async function(req, res, next) { @@ -367,15 +348,9 @@ router.post('/:projectUrl/user/del', async function(req, res, next) { return res.json({ error: "you don't have permission to manage project settings and users" }); } - model.projects.removeUser(req.body.user_id, req.project.project_id, async function(err, result){ - if(err) return next(err); - - debug("remove user:", result); - if (rfcxConfig.coreAPIEnabled) { - await model.projects.removeUserRoleInCoreAPI(req.body.user_id, req.project.project_id, req.session.idToken) - } + model.projects.updateUserRoleInArbimonAndCoreAPI({user_id: req.body.user_id, project_id: req.project.project_id}, req.session.idToken, 'remove').then(function() { res.json({ success: true }); - }); + }).catch(next); }); router.post('/:projectUrl/remove', function(req, res, next) { From cc94bb4771904cf50be1407eceaf528c9cea90ed Mon Sep 17 00:00:00 2001 From: Evgeniia Rassokhina Date: Tue, 13 Jul 2021 23:16:37 +0300 Subject: [PATCH 03/15] code is improved --- app/model/projects.js | 174 +++++++++++---------------- app/model/sites.js | 33 +++-- app/routes/data-api/ingest.js | 3 +- app/routes/data-api/integration.js | 3 +- app/routes/data-api/project/sites.js | 1 - 5 files changed, 94 insertions(+), 120 deletions(-) diff --git a/app/model/projects.js b/app/model/projects.js index 69393f19b..1848963b2 100644 --- a/app/model/projects.js +++ b/app/model/projects.js @@ -250,9 +250,8 @@ var Projects = { * @param {Number} project.owner_id - creator id * @param {Number} project.project_type_id * @param {Boolean} project.is_private - * @param {Function} callback(err, projectId) */ - create: async function(project, owner_id, db, callback) { + create: async function(project, owner_id, db) { var schema = joi.object().keys({ name: joi.string(), url: joi.string(), @@ -286,107 +285,69 @@ var Projects = { project.processing_usage = 0; project.pattern_matching_enabled = 1; - var q = 'INSERT INTO projects \n'+ - 'SET ?'; - - var q2 = 'INSERT INTO user_project_role \n'+ - 'SET ?'; - - var createPlan = 'INSERT INTO project_plans \n'+ - 'SET ?'; - - var updatePlan = 'UPDATE projects \n'+ - 'SET current_plan = ? \n'+ - 'WHERE project_id = ?'; - var projectId; - if (db) { - - async.waterfall([ - function insertProject(cb) { - db.query(q, project, cb); - }, - function insertOwner(result, fields, cb) { - projectId = result.insertId; - - var values = { - user_id: owner_id, - project_id: projectId, - role_id: 4 // owner role id - }; - - db.query(q2, values, cb); - }, - function insertPlan(result, fields, cb) { - plan.project_id = projectId; - plan.created_on = new Date(); - - db.query(createPlan, plan, cb); - }, - function updateCurrentPlan(result, fields, cb) { - db.query(updatePlan, [result.insertId, projectId], cb); - }, - function commit(result, fields, cb) { - db.commit(cb); - } - ], - function(err) { - if(err) { - callback(err); - return; - } - callback(null, projectId); - }); + if (db) { + projectId = await this.summaryFunctions(db, project, owner_id, plan); + return projectId; } else { - dbpool.getConnection(function(err, db) { - db.beginTransaction(function(err){ - if(err) return callback(err); - - async.waterfall([ - function insertProject(cb) { - db.query(q, project, cb); - }, - function insertOwner(result, fields, cb) { - projectId = result.insertId; - - var values = { - user_id: owner_id, - project_id: projectId, - role_id: 4 // owner role id - }; - - db.query(q2, values, cb); - }, - function insertPlan(result, fields, cb) { - plan.project_id = projectId; - plan.created_on = new Date(); - - db.query(createPlan, plan, cb); - }, - function updateCurrentPlan(result, fields, cb) { - db.query(updatePlan, [result.insertId, projectId], cb); - }, - function commit(result, fields, cb) { - db.commit(cb); - } - ], - function(err) { + dbpool.getConnection(function(err, db) { + db.beginTransaction(async function(err){ + if(err) return callback(err); + db.release(); - if(err) { + + projectId = await this.summaryFunctions(db, project, owner_id, plan); + + if (!result) { db.rollback(function() { callback(err); }); return; } - callback(null, projectId); + return projectId; }); }); - }); } }, + insertProjectAsync: async function(project, connection) { + let insert = util.promisify(connection.query); + return insert('INSERT INTO projects SET ?', project); + }, + + insertOwnerAsync: async function(values, connection) { + let insert = util.promisify(connection.query); + return insert('INSERT INTO user_project_role SET ?', values); + }, + + insertPlanAsync: async function(plan, connection) { + let insert = util.promisify(connection.query); + return insert('INSERT INTO project_plans SET ?', plan); + }, + + updateCurrentPlanAsync: async function(result, projectId, connection) { + let insert = util.promisify(connection.query); + return insert('UPDATE projects SET current_plan = ? WHERE project_id = ?', [result.insertId, projectId]); + }, + + summaryFunctions: async function(db, project, owner_id, plan) { + let result = await this.insertProjectAsync(project, db); + let projectId = result.insertId; + let values = { + user_id: owner_id, + project_id: projectId, + role_id: 4 + }; + await this.insertOwnerAsync(values, db); + plan.project_id = projectId; + plan.created_on = new Date(); + let newPlan = await this.insertPlanAsync(plan, db); + await this.updateCurrentPlanAsync(newPlan, projectId, db); + return projectId; + }, + /** * updates a project. * @param {Object} project @@ -439,13 +400,13 @@ var Projects = { delete projectInfo.plan; return q.all([ - db.query( + (db? db.query : dbpool.query)( 'UPDATE projects\n'+ 'SET ?\n'+ 'WHERE project_id = ?', [ projectInfo, projectId ]), - projectInfoPlan && db.query( + projectInfoPlan && (db? db.query : dbpool.query)( "UPDATE project_plans\n"+ "SET ?\n" + "WHERE project_id=?", [ @@ -987,8 +948,8 @@ var Projects = { .then(async (con) => { connection = con; await connection.beginTransaction(); - let newProject = await this.createProject(project, userId, connection); - project.project_id = newProject.project_id; + let newProjectId = await this.createProject(project, userId, connection); + project.project_id = newProjectId; if (rfcxConfig.coreAPIEnabled) { let externalProjectId = await this.createInCoreAPI(project, token); await this.setExternalId(project.project_id, externalProjectId, connection); @@ -1001,8 +962,8 @@ var Projects = { if (connection) { await connection.rollback(); await connection.release(); - throw new APIError('Failed to create project'); } + throw new APIError('Failed to create project'); }) }, @@ -1054,8 +1015,8 @@ var Projects = { if (connection) { await connection.rollback(); await connection.release(); - throw new APIError('Failed to update project'); } + throw new APIError('Failed to update project'); }) }, @@ -1076,9 +1037,12 @@ var Projects = { body: JSON.stringify(body) } return rp(options).then((response) => { - if (response.statusCode === 403) { - throw new Error('Forbidden error.') - } + try { + const body = JSON.parse(response.body); + if (body && body.error) { + throw new APIError('Failed to update project'); + } + } catch (e) { } }) }, @@ -1095,9 +1059,12 @@ var Projects = { body: JSON.stringify(body) } return rp(options).then((response) => { - if (response.statusCode === 403) { - throw new Error('Forbidden error.') - } + try { + const body = JSON.parse(response.body); + if (body && body.error) { + throw new APIError('Failed to delete project'); + } + } catch (e) { } }) }, @@ -1162,9 +1129,8 @@ var Projects = { stripUnknown: true, presence: 'required', }); - // const id = await this.create(projectData, userId, connection); - const id = await q.ninvoke(this, "create", projectData, userId, connection); - return this.find({ id }).get(0) + const id = await this.create(projectData, userId, connection); + return id; }, removeProject: async function(options) { @@ -1185,8 +1151,8 @@ var Projects = { if (db) { await db.rollback(); await db.release(); - throw new APIError('Failed to delete project', 422); } + throw new APIError('Failed to delete project'); }) }, diff --git a/app/model/sites.js b/app/model/sites.js index 39c99763f..daa415792 100644 --- a/app/model/sites.js +++ b/app/model/sites.js @@ -683,8 +683,8 @@ var Sites = { if (connection) { await connection.rollback(); await connection.release(); - throw new APIError('Failed to create site'); } + throw new APIError('Failed to create site'); }) }, @@ -727,7 +727,6 @@ var Sites = { db = connection; await db.beginTransaction(); await this.updateAsync(site, connection); - let personalProject = await model.projects.findOrCreatePersonalProject(user); if (rfcxConfig.coreAPIEnabled) { await this.updateInCoreAPI({ site_id: site.site_id, @@ -735,19 +734,19 @@ var Sites = { lat: site.lat, lon: site.lon, alt: site.alt, - ...personalProject && personalProject.project_id === site.project_id ? {} : { project_id: site.project_id } + project_id: site.project_id }, idToken) }; await db.commit(); await db.release(); }) .catch(async (err) => { - console.log('Failed to update site', err); + console.log('err', err); if (db) { await db.rollback(); await db.release(); - throw new APIError('Failed to update site'); } + throw new APIError('Failed to update site'); }) }, @@ -769,9 +768,12 @@ var Sites = { body: JSON.stringify(body) } return rp(options).then((response) => { - if (response.statusCode === 403) { - throw new Error('Forbidden error.') - } + try { + const body = JSON.parse(response.body); + if (body && body.error) { + throw new APIError('Failed to update site'); + } + } catch (e) { } }) }, @@ -789,12 +791,12 @@ var Sites = { await db.release(); }) .catch(async (err) => { - console.log('Failed to delete site', err); + console.log('err', err); if (db) { await db.rollback(); await db.release(); - throw new APIError('Failed to delete site'); } + throw new APIError('Failed to delete site'); }) }, @@ -809,9 +811,14 @@ var Sites = { } } return rp(options).then((response) => { - if (response.statusCode === 403) { - throw new Error('Forbidden error.') - } + console.log('response', response) + try { + const body = JSON.parse(response.body); + console.log('body', body) + if (body && body.error) { + throw new APIError('Failed to delete site'); + } + } catch (e) { } }) }, diff --git a/app/routes/data-api/ingest.js b/app/routes/data-api/ingest.js index 13fb2634a..ea40323ca 100644 --- a/app/routes/data-api/ingest.js +++ b/app/routes/data-api/ingest.js @@ -50,13 +50,14 @@ router.post('/recordings/create', verifyToken(), hasRole(['systemUser']), async const user = (await model.users.findByEmailAsync('support@rfcx.org'))[0]; // Create missing project const url = await model.projects.findUniqueUrl(externalProject.name, externalProject.id, user.user_id) - project = await model.projects.createProject({ + let projectId = await model.projects.createProject({ name: externalProject.name, description: externalProject.description, is_private: true, external_id: externalProject.id, url: externalSite.site.guid }, user.user_id) + project = await model.projects.find({ projectId }).get(0); } // Create missing site const siteInsertData = await model.sites.insertAsync({ diff --git a/app/routes/data-api/integration.js b/app/routes/data-api/integration.js index e9685392d..c5a683ac3 100644 --- a/app/routes/data-api/integration.js +++ b/app/routes/data-api/integration.js @@ -21,7 +21,8 @@ router.post('/projects', verifyToken(), hasRole(['appUser', 'rfcxUser']), async const user = await model.users.ensureUserExistFromAuth0(req.user); const url = await model.projects.findUniqueUrl(params.name, params.external_id, user.user_id) const { name, description, is_private, external_id } = params - const project = await model.projects.createProject({ name, description, is_private, external_id, url }, user.user_id) + const projectId = await model.projects.createProject({ name, description, is_private, external_id, url }, user.user_id); + const project = await model.projects.find({ projectId }).get(0); res.status(201).json(project); } catch (e) { httpErrorHandler(req, res, 'Failed creating a site')(e); diff --git a/app/routes/data-api/project/sites.js b/app/routes/data-api/project/sites.js index aa1b326d0..1a991918e 100644 --- a/app/routes/data-api/project/sites.js +++ b/app/routes/data-api/project/sites.js @@ -44,7 +44,6 @@ router.post('/create', function(req, res, next) { catch(e) { return next(err); } - res.json({ message: "New site created" }); }); }); From 71e1237f396f8b06d6c84ff7952695a3cff43281 Mon Sep 17 00:00:00 2001 From: RatreeOchn Date: Tue, 20 Jul 2021 12:37:41 +0700 Subject: [PATCH 04/15] Not update the value of present when user review from PM page (default null) --- app/model/recordings.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/model/recordings.js b/app/model/recordings.js index b1f1e9a73..a421a6fd3 100644 --- a/app/model/recordings.js +++ b/app/model/recordings.js @@ -786,8 +786,8 @@ var Recordings = { if (validation.determinedFrom == 'patternMatching') { queryHandler( "INSERT INTO recording_validations(recording_id, user_id, species_id, songtype_id, present, project_id, present_review) \n" + - " VALUES (" + dbpool.escape([valobj.recording, valobj.user, valobj.species, valobj.songtype, valobj.val, valobj.project_id, valobj.val]) + ") \n" + - " ON DUPLICATE KEY UPDATE present = VALUES(present), present_review = CASE WHEN present = 0 AND present_review = 0 THEN 0 WHEN present = 0 THEN present_review - 1 ELSE present_review + 1 END", function(err, data){ + " VALUES (" + dbpool.escape([valobj.recording, valobj.user, valobj.species, valobj.songtype, null, valobj.project_id, valobj.val]) + ") \n" + + " ON DUPLICATE KEY UPDATE present_review = CASE WHEN "+ dbpool.escape(valobj.val)+ " = 0 AND present_review = 0 THEN 0 WHEN "+ dbpool.escape(valobj.val)+ " = 0 THEN present_review - 1 ELSE present_review + 1 END", function(err, data){ if (err) { callback(err); return; } callback(null, valobj); }); From 3e020e3fbe13a361d80c6d4190becd1be0f1d8b6 Mon Sep 17 00:00:00 2001 From: RatreeOchn Date: Tue, 20 Jul 2021 14:14:11 +0700 Subject: [PATCH 05/15] Fix should not go from 1 to 2 when we change a PM result from present to present --- app/model/pattern_matchings.js | 9 +++ .../data-api/project/pattern_matchings.js | 63 +++++++++++-------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/app/model/pattern_matchings.js b/app/model/pattern_matchings.js index de6714b44..4d5fefe33 100644 --- a/app/model/pattern_matchings.js +++ b/app/model/pattern_matchings.js @@ -640,6 +640,15 @@ var PatternMatchings = { rois, ]) : Promise.resolve(); }, + + getRoi(patternMatchingId, roisId){ + return dbpool.query( + "SELECT *\n" + + "FROM pattern_matching_rois\n" + + "WHERE pattern_matching_id = ? AND pattern_matching_roi_id IN (?)", [ + patternMatchingId, roisId + ]); + }, getCountRoisMatchByAttr(patternMatchingId, recordingId, validation){ return dbpool.query( diff --git a/app/routes/data-api/project/pattern_matchings.js b/app/routes/data-api/project/pattern_matchings.js index b0843c821..2b5d8974e 100644 --- a/app/routes/data-api/project/pattern_matchings.js +++ b/app/routes/data-api/project/pattern_matchings.js @@ -180,34 +180,43 @@ router.get('/:patternMatching/audio/:roiId', function(req, res, next) { router.post('/:patternMatching/validate', function(req, res, next) { res.type('json'); - model.patternMatchings.validateRois(req.params.patternMatching, req.body.rois, req.body.validation) - .then(function(rois){ - if (req.body.rois && req.body.rois.length) { - return model.patternMatchings.getPatternMatchingRois({rois: req.body.rois}).then(async function(rois) { - for (let roi of rois) { - var validation = 2 - if(req.body.validation != null) { - validation = req.body.validation - } - // Save validated rois in the recording validations table if the roi is validated; - // Remove the recording validation row if the roi is absent or not validated and, the row exists. - await model.recordings.validate( - {id: roi.recording_id}, - req.session.user.id, - req.project.project_id, - { class: `${roi.species_id}-${roi.songtype_id}`, val: validation, determinedFrom: 'patternMatching'}, - function(err, validations) { - if(err) return next(err); - return validations; - }) - } - }) + var roiList = [] + model.patternMatchings.getRoi(req.params.patternMatching, req.body.rois).then(function(rois){ + for (let roi of rois) { + if (roi.validated != req.body.validation) { + roiList.push(roi.pattern_matching_roi_id) } - }).then(function(rois) { - res.json({ - rois: req.body.rois, - validation: req.body.validation, - }); + } + + model.patternMatchings.validateRois(req.params.patternMatching, roiList, req.body.validation) + .then(function(rois){ + if (roiList && roiList.length) { + return model.patternMatchings.getPatternMatchingRois({rois: roiList}).then(async function(rois) { + for (let roi of rois) { + var validation = 2 + if(req.body.validation != null) { + validation = req.body.validation + } + // Save validated rois in the recording validations table if the roi is validated; + // Remove the recording validation row if the roi is absent or not validated and, the row exists. + await model.recordings.validate( + {id: roi.recording_id}, + req.session.user.id, + req.project.project_id, + { class: `${roi.species_id}-${roi.songtype_id}`, val: validation, determinedFrom: 'patternMatching'}, + function(err, validations) { + if(err) return next(err); + return validations; + }) + } + }) + } + }).then(function(rois) { + res.json({ + rois: roiList, + validation: req.body.validation, + }); + }).catch(next); }).catch(next); }); From d2cae36e91b37c8fac2b63d19dbb190763f4f519 Mon Sep 17 00:00:00 2001 From: RatreeOchn Date: Tue, 20 Jul 2021 21:58:16 +0700 Subject: [PATCH 06/15] =?UTF-8?q?Fix=20UI=20for=20show=20=E2=80=9CPresent?= =?UTF-8?q?=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- assets/app/app/visualizer/validator-main.html | 4 ++-- assets/app/app/visualizer/validator.js | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/assets/app/app/visualizer/validator-main.html b/assets/app/app/visualizer/validator-main.html index f98f9b7a1..2772c0eaa 100644 --- a/assets/app/app/visualizer/validator-main.html +++ b/assets/app/app/visualizer/validator-main.html @@ -21,11 +21,11 @@ {{ cls.songtype_name }}
-
+
{{val_state(cls.id).label}}
-