Merge pull request #1014 from rfcx/bug/update-getting-statistics-on-t…

…he-homepage

Fixed the issue about long loading website metrics on the Homepage

CHANGELOG.md

@@ -1,11 +1,12 @@
 # Arbimon Release Notes
 
-## v3.0.59 - August 08, 2022
+## v3.0.59 - August 10, 2022
 
 Resolved issues:
 
 - Removed `UMAP` clustering visualization type
 - Show 0 frequency in the AED pop-up, if the box frequency equals to 0
+- Fixed the issue about long loading website metrics on the Homepage
 
 ## v3.0.56 - June 24, 2022
 

DEPLOYMENT_NOTES.md

@@ -1,5 +1,9 @@
 # Arbimon Deployment Notes
 
+## v3.0.58
+
+- Run 031-cached-metrics-table.sql on the production
+
 ## v3.0.55
 
 - Run 028-add-created-at-updated-at-to-projects.sql on the staging/production sides

app/model/jobs.js

@@ -483,8 +483,9 @@ var Jobs = {
         return null;
     },
 
-    countAllCompletedJobs: function() {
-        return dbpool.query("SELECT count(*) AS count FROM jobs WHERE state='completed'");
+    countAllCompletedJobs: async function() {
+        const q = "SELECT count(*) AS count FROM jobs WHERE state='completed'"
+        return dbpool.query(q).get(0).get('count')
     },
 
     /** Computes a summary of the current jobs status, by job type.

app/model/projects.js

@@ -43,11 +43,24 @@ var Projects = {
         }
     },
 
-    countAllProjects: function(callback) {
-        var q = 'SELECT count(*) AS count \n'+
-                'FROM `projects`';
+    countAllProjects: async function() {
+        const q = 'SELECT count(*) AS count FROM projects';
 
-        queryHandler(q, callback);
+        return dbpool.query(q).get(0).get('count')
+    },
+
+    getCachedMetrics: async function(key) {
+        const q = `SELECT * FROM cached_metrics cm
+                 WHERE cm.key = '${key}'`
+
+        return dbpool.query(q)
+    },
+
+    updateCachedMetrics: async function(opts) {
+        const q = `UPDATE cached_metrics cm
+                SET cm.value = ${opts.value}, cm.expires_at = '${opts.expires_at}'
+                WHERE cm.key = '${opts.key}'`
+        return dbpool.query(q)
     },
 
     listAll: function(callback) {

app/model/recordings.js

@@ -1772,11 +1772,15 @@ var Recordings = {
     },
 
     countAllSpecies: function() {
-        return dbpool.query('SELECT COUNT(DISTINCT species_id) AS count FROM recording_validations WHERE present = 1 OR present_review > 0 OR present_aed > 0');
+        const q = 'SELECT COUNT(DISTINCT species_id) AS count FROM recording_validations WHERE present = 1 OR present_review > 0 OR present_aed > 0'
+
+        return dbpool.query(q).get(0).get('count')
     },
 
     countAllRecordings: function() {
-        return dbpool.query('SELECT count(*) AS count FROM recordings');
+        const q = 'SELECT count(*) AS count FROM recordings'
+
+        return dbpool.query(q).get(0).get('count')
     },
 
     /* fetch count of project recordings.

app/routes/data-api/project/index.js

@@ -14,25 +14,7 @@ var config = require('../../../config');
 const rfcxConfig = config('rfcx');
 const csv_stringify = require('csv-stringify');
 const dayInMs = 24 * 60 * 60 * 1000;
-
-let summaryData = {
-    projects: {
-        count: 0,
-        time: Date.now()
-    },
-    species: {
-        count: 0,
-        time: Date.now()
-    },
-    rec: {
-        count: 0,
-        time: Date.now()
-    },
-    jobs: {
-        count: 0,
-        time: Date.now()
-    }
-};
+const moment = require('moment');
 
 var model = require('../../../model');
 
@@ -117,58 +99,73 @@ router.get('/:projectUrl/info/source-project', function(req, res, next) {
 
 });
 
+const getCountForSelectedMetric = async function(key) {
+    let count
+    switch (key) {
+        case 'project-count':
+            count = await model.projects.countAllProjects()
+            break;
+        case 'job-count':
+            count = await model.jobs.countAllCompletedJobs()
+            break;
+        case 'species-count':
+            count = await model.recordings.countAllSpecies()
+            break;
+        case 'recordings-count':
+            count = await model.recordings.countAllRecordings()
+            break;
+    }
+    return count
+}
+
+const getRandomMin = function(max, min) {
+    return Math.floor(Math.random() * (max - min + 1) + min);
+}
+
+const getCachedMetrics = async function(req, res, key, next) {
+    model.projects.getCachedMetrics(key).then(async function(results) {
+        if (!results.length) return
+        const [result] = results
+        const count = result.value
+
+        res.json(count)
+
+        const dateNow = moment.utc().valueOf()
+        const dateIndb = moment.utc(result.expires_at).valueOf()
+        // Recalculate metrics each day and save the results in the db
+        if (dateNow > dateIndb) {
+            const value = await getCountForSelectedMetric(key)
+            const expires_at = moment.utc().add(1, 'days').add(getRandomMin(0, 60), 'minutes').format('YYYY-MM-DD HH:mm:ss')
+            await model.projects.updateCachedMetrics({ key, value, expires_at })
+        }
+    }).catch(next);
+}
+
 router.get('/projects-count', function(req, res, next) {
     res.type('json');
-
-    if (summaryData.projects.count === 0 || (Date.now() - summaryData.projects.time > dayInMs)) {
-        model.projects.countAllProjects(function(err, results) {
-            if(err) return next(err);
-            summaryData.projects.count = results[0].count;
-            res.json(results[0].count);
-        });
-    }
-    else {
-        return res.json(summaryData.projects.count);
-    }
+    const key = 'project-count'
+    getCachedMetrics(req, res, key, next);
 });
 
 router.get('/jobs-count', function(req, res, next) {
     res.type('json');
-    if (summaryData.jobs.count === 0 || (Date.now() - summaryData.jobs.time > dayInMs)) {
-        model.jobs.countAllCompletedJobs().then((results) => {
-            summaryData.jobs.count = results[0].count;
-            res.json(results[0].count);
-        }).catch(next);
-    }
-    else {
-        return res.json(summaryData.jobs.count);
-    }
+    const key = 'job-count'
+
+    getCachedMetrics(req, res, key, next);
 });
 
 router.get('/recordings-species-count', function(req, res, next) {
     res.type('json');
-    if (summaryData.species.count === 0 || (Date.now() - summaryData.species.time > dayInMs)) {
-        model.recordings.countAllSpecies().then((results) => {
-            summaryData.species.count = results[0].count;
-            res.json(results[0].count);
-        }).catch(next);
-    }
-    else {
-        return res.json(summaryData.species.count);
-    }
+    const key = 'species-count'
+
+    getCachedMetrics(req, res, key, next);
 });
 
 router.get('/recordings-count', function(req, res, next) {
     res.type('json');
-    if (summaryData.rec.count === 0 || (Date.now() - summaryData.rec.time > dayInMs)) {
-        model.recordings.countAllRecordings().then((results) => {
-            summaryData.rec.count = results[0].count;
-            res.json(results[0].count);
-        }).catch(next);
-    }
-    else {
-        return res.json(summaryData.rec.count);
-    }
+    const key = 'recordings-count'
+
+    getCachedMetrics(req, res, key, next);
 });
 
 router.post('/:projectUrl/info/update', function(req, res, next) {

scripts/db/031-cached-metrics-table.sql

@@ -0,0 +1,7 @@
+CREATE TABLE `cached_metrics` (
+  `key` VARCHAR(20) NOT NULL,
+  `value` INT NOT NULL DEFAULT 0,
+  `expires_at` DATETIME NOT NULL,
+  PRIMARY KEY (`key`),
+  UNIQUE KEY `unique_key` (`key`)
+);