From 5e0a3a2b87f9dfcda945689ee874bd60184d61e2 Mon Sep 17 00:00:00 2001 From: Evgeniia Rassokhina Date: Tue, 9 Aug 2022 01:36:18 +0300 Subject: [PATCH 1/3] Fixed the issue about long loading website metrics on the Homepage --- CHANGELOG.md | 2 + app/model/jobs.js | 5 +- app/model/projects.js | 21 ++++-- app/model/recordings.js | 8 ++- app/routes/data-api/project/index.js | 89 ++++++++++++++----------- scripts/db/031-cached-metrics-table.sql | 7 ++ 6 files changed, 86 insertions(+), 46 deletions(-) create mode 100644 scripts/db/031-cached-metrics-table.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index f532cde3b..a38ce041f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Arbimon Release Notes +- Fixed the issue about long loading website metrics on the Homepage + ## v3.0.56 - June 24, 2022 Resolved issues: diff --git a/app/model/jobs.js b/app/model/jobs.js index 34e173538..4871bda4b 100644 --- a/app/model/jobs.js +++ b/app/model/jobs.js @@ -483,8 +483,9 @@ var Jobs = { return null; }, - countAllCompletedJobs: function() { - return dbpool.query("SELECT count(*) AS count FROM jobs WHERE state='completed'"); + countAllCompletedJobs: async function() { + const q = "SELECT count(*) AS count FROM jobs WHERE state='completed'" + return dbpool.query(q).get(0).get('count') }, /** Computes a summary of the current jobs status, by job type. diff --git a/app/model/projects.js b/app/model/projects.js index 3cb500ee1..601eb040e 100644 --- a/app/model/projects.js +++ b/app/model/projects.js @@ -43,11 +43,24 @@ var Projects = { } }, - countAllProjects: function(callback) { - var q = 'SELECT count(*) AS count \n'+ - 'FROM `projects`'; + countAllProjects: async function() { + const q = 'SELECT count(*) AS count FROM projects'; - queryHandler(q, callback); + return dbpool.query(q).get(0).get('count') + }, + + getCachedMetrics: async function(key) { + const q = `SELECT * FROM cached_metrics cm + WHERE cm.key = '${key}'` + + return dbpool.query(q) + }, + + updateCachedMetrics: async function(opts) { + const q = `UPDATE cached_metrics cm + SET cm.value = ${opts.value}, cm.expires_at = '${opts.expires_at}' + WHERE cm.key = '${opts.key}'` + return dbpool.query(q) }, listAll: function(callback) { diff --git a/app/model/recordings.js b/app/model/recordings.js index 9483286b9..9b799e874 100644 --- a/app/model/recordings.js +++ b/app/model/recordings.js @@ -1772,11 +1772,15 @@ var Recordings = { }, countAllSpecies: function() { - return dbpool.query('SELECT COUNT(DISTINCT species_id) AS count FROM recording_validations WHERE present = 1 OR present_review > 0 OR present_aed > 0'); + const q = 'SELECT COUNT(DISTINCT species_id) AS count FROM recording_validations WHERE present = 1 OR present_review > 0 OR present_aed > 0' + + return dbpool.query(q).get(0).get('count') }, countAllRecordings: function() { - return dbpool.query('SELECT count(*) AS count FROM recordings'); + const q = 'SELECT count(*) AS count FROM recordings' + + return dbpool.query(q).get(0).get('count') }, /* fetch count of project recordings. diff --git a/app/routes/data-api/project/index.js b/app/routes/data-api/project/index.js index 44ffd6b3e..d690deeac 100644 --- a/app/routes/data-api/project/index.js +++ b/app/routes/data-api/project/index.js @@ -14,6 +14,7 @@ var config = require('../../../config'); const rfcxConfig = config('rfcx'); const csv_stringify = require('csv-stringify'); const dayInMs = 24 * 60 * 60 * 1000; +const moment = require('moment'); let summaryData = { projects: { @@ -117,58 +118,70 @@ router.get('/:projectUrl/info/source-project', function(req, res, next) { }); +const getCountForSelectedMetric = async function(key) { + let count + switch (key) { + case 'project-count': + count = await model.projects.countAllProjects() + break; + case 'job-count': + count = await model.jobs.countAllCompletedJobs() + break; + case 'species-count': + count = await model.recordings.countAllSpecies() + break; + case 'recordings-count': + count = await model.recordings.countAllRecordings() + break; + } + return count +} + +const getRandomMinAsMs = function(max, min) { + return Math.floor(Math.random() * (max - min + 1) + min); +} + +const getCachedMetrics = async function(req, res, opts, next) { + model.projects.getCachedMetrics(opts.key).then(async function(results) { + if (!results.length) return + const [result] = results + const count = result.value + const dateNow = moment.utc().valueOf() + const dateIndb = moment.utc(result.expires_at).valueOf() + if (dateNow > dateIndb) { + const value = await getCountForSelectedMetric(opts.key) + const expires_at = moment.utc(dateNow + dayInMs + getRandomMinAsMs(0, 60000)).format('YYYY-MM-DD HH:mm:ss') + await model.projects.updateCachedMetrics({ ...opts, value, expires_at }) + } + return res.json(count); + }).catch(next); +} + router.get('/projects-count', function(req, res, next) { res.type('json'); - - if (summaryData.projects.count === 0 || (Date.now() - summaryData.projects.time > dayInMs)) { - model.projects.countAllProjects(function(err, results) { - if(err) return next(err); - summaryData.projects.count = results[0].count; - res.json(results[0].count); - }); - } - else { - return res.json(summaryData.projects.count); - } + const opts = { key: 'project-count' } + getCachedMetrics(req, res, opts, next); }); router.get('/jobs-count', function(req, res, next) { res.type('json'); - if (summaryData.jobs.count === 0 || (Date.now() - summaryData.jobs.time > dayInMs)) { - model.jobs.countAllCompletedJobs().then((results) => { - summaryData.jobs.count = results[0].count; - res.json(results[0].count); - }).catch(next); - } - else { - return res.json(summaryData.jobs.count); - } + const opts = { key: 'job-count' } + + getCachedMetrics(req, res, opts, next); }); router.get('/recordings-species-count', function(req, res, next) { res.type('json'); - if (summaryData.species.count === 0 || (Date.now() - summaryData.species.time > dayInMs)) { - model.recordings.countAllSpecies().then((results) => { - summaryData.species.count = results[0].count; - res.json(results[0].count); - }).catch(next); - } - else { - return res.json(summaryData.species.count); - } + const opts = { key: 'species-count' } + + getCachedMetrics(req, res, opts, next); }); router.get('/recordings-count', function(req, res, next) { res.type('json'); - if (summaryData.rec.count === 0 || (Date.now() - summaryData.rec.time > dayInMs)) { - model.recordings.countAllRecordings().then((results) => { - summaryData.rec.count = results[0].count; - res.json(results[0].count); - }).catch(next); - } - else { - return res.json(summaryData.rec.count); - } + const opts = { key: 'recordings-count' } + + getCachedMetrics(req, res, opts, next); }); router.post('/:projectUrl/info/update', function(req, res, next) { diff --git a/scripts/db/031-cached-metrics-table.sql b/scripts/db/031-cached-metrics-table.sql new file mode 100644 index 000000000..77cdee16b --- /dev/null +++ b/scripts/db/031-cached-metrics-table.sql @@ -0,0 +1,7 @@ +CREATE TABLE `cached_metrics` ( + `key` VARCHAR(20) NOT NULL, + `value` INT NOT NULL DEFAULT 0, + `expires_at` DATETIME NOT NULL, + PRIMARY KEY (`key`), + UNIQUE KEY `unique_key` (`key`) +); From 2426701bd166eaf57fb259a91630340c22014265 Mon Sep 17 00:00:00 2001 From: Evgeniia Rassokhina Date: Tue, 9 Aug 2022 01:40:15 +0300 Subject: [PATCH 2/3] Update deployment notes --- DEPLOYMENT_NOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DEPLOYMENT_NOTES.md b/DEPLOYMENT_NOTES.md index a609e16dd..5f684c73a 100644 --- a/DEPLOYMENT_NOTES.md +++ b/DEPLOYMENT_NOTES.md @@ -1,5 +1,9 @@ # Arbimon Deployment Notes +## v3.0.58 + +- Run 031-cached-metrics-table.sql on the production + ## v3.0.55 - Run 028-add-created-at-updated-at-to-projects.sql on the staging/production sides From aae237e2c7fd0693e6108e0f1ae895c0a187447c Mon Sep 17 00:00:00 2001 From: Evgeniia Rassokhina Date: Tue, 9 Aug 2022 12:32:46 +0300 Subject: [PATCH 3/3] Fix after reviewing the task --- app/routes/data-api/project/index.js | 52 ++++++++++------------------ 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/app/routes/data-api/project/index.js b/app/routes/data-api/project/index.js index d690deeac..d514d1bf3 100644 --- a/app/routes/data-api/project/index.js +++ b/app/routes/data-api/project/index.js @@ -16,25 +16,6 @@ const csv_stringify = require('csv-stringify'); const dayInMs = 24 * 60 * 60 * 1000; const moment = require('moment'); -let summaryData = { - projects: { - count: 0, - time: Date.now() - }, - species: { - count: 0, - time: Date.now() - }, - rec: { - count: 0, - time: Date.now() - }, - jobs: { - count: 0, - time: Date.now() - } -}; - var model = require('../../../model'); // routes @@ -137,51 +118,54 @@ const getCountForSelectedMetric = async function(key) { return count } -const getRandomMinAsMs = function(max, min) { +const getRandomMin = function(max, min) { return Math.floor(Math.random() * (max - min + 1) + min); } -const getCachedMetrics = async function(req, res, opts, next) { - model.projects.getCachedMetrics(opts.key).then(async function(results) { +const getCachedMetrics = async function(req, res, key, next) { + model.projects.getCachedMetrics(key).then(async function(results) { if (!results.length) return const [result] = results const count = result.value + + res.json(count) + const dateNow = moment.utc().valueOf() const dateIndb = moment.utc(result.expires_at).valueOf() + // Recalculate metrics each day and save the results in the db if (dateNow > dateIndb) { - const value = await getCountForSelectedMetric(opts.key) - const expires_at = moment.utc(dateNow + dayInMs + getRandomMinAsMs(0, 60000)).format('YYYY-MM-DD HH:mm:ss') - await model.projects.updateCachedMetrics({ ...opts, value, expires_at }) + const value = await getCountForSelectedMetric(key) + const expires_at = moment.utc().add(1, 'days').add(getRandomMin(0, 60), 'minutes').format('YYYY-MM-DD HH:mm:ss') + await model.projects.updateCachedMetrics({ key, value, expires_at }) } - return res.json(count); }).catch(next); } router.get('/projects-count', function(req, res, next) { res.type('json'); - const opts = { key: 'project-count' } - getCachedMetrics(req, res, opts, next); + const key = 'project-count' + getCachedMetrics(req, res, key, next); }); router.get('/jobs-count', function(req, res, next) { res.type('json'); - const opts = { key: 'job-count' } + const key = 'job-count' - getCachedMetrics(req, res, opts, next); + getCachedMetrics(req, res, key, next); }); router.get('/recordings-species-count', function(req, res, next) { res.type('json'); - const opts = { key: 'species-count' } + const key = 'species-count' - getCachedMetrics(req, res, opts, next); + getCachedMetrics(req, res, key, next); }); router.get('/recordings-count', function(req, res, next) { res.type('json'); - const opts = { key: 'recordings-count' } + const key = 'recordings-count' - getCachedMetrics(req, res, opts, next); + getCachedMetrics(req, res, key, next); }); router.post('/:projectUrl/info/update', function(req, res, next) {