Skip to content

Commit

Permalink
Merge pull request #715 from rfcx/bug/CE-1067-slow-recordings-query
Browse files Browse the repository at this point in the history 
CE-1067, CE-1070, CE-1063 slow queries
  • Loading branch information
@rassokhin-s
rassokhin-s authored Jul 30, 2021
2 parents 6045367 + 8ccc8c1 commit 0fbcf49
Show file tree
Hide file tree
Showing 7 changed files with 28 additions and 139 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ Performance improvements:
- CE-988 The datime_local column is removed
- CE-1143 Update "Unvalidated", "All", "Present", "Not present", "200 top scores per site", "Score per site" filters on Pattern Matching details page to return results per site
- CE-1158 update PM filters logic so "Best per Site" and "Best per Site, Day" return results by 20 sites batches
- CE-1067 Speed up recordings queries which search by selected sites on Recordings and Visualizer pages
- CE-1070 Speed up recordings total count endpoint
- CE-1063 Delete project stats bar from Summary page

## v3.0.32 - June 26, 2021

Expand Down
9 changes: 9 additions & 0 deletions TEST_NOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,15 @@ Test Notes are used to list what pages / components / features / user flows are
- Check that pagination works correctly for these two filters
- Check that other filters still work correctly
- Check that switching between filters, pages, sites, etc works correctly
- CE-1067 Speed up recordings queries which search by selected sites on Recordings and Visualizer pages
- Check that filters on Recordings page still work correctly
- Check that pagination on Recordings page still work correctly
- Check that site list and recordings list on Visualizer page still works correctly
- CE-1070 Speed up recordings total count endpoint
- Check that total recordings count is still correctly calculated on the homepage
- Check that total recordings count is still correctly calculated on Project summary page
- CE-1063 Delete project stats bar from Summary page
- Check that summary page still works correctly

## v3.0.32

Expand Down
2 changes: 1 addition & 1 deletion app/model/projects.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -933,7 +933,7 @@ var Projects = {

// this includes recordings processing
totalRecordings: function(project_id, callback) {
var q = "SELECT count(*) as count \n" +
var q = "SELECT count(recording_id) as count \n" +
"FROM recordings AS r JOIN sites AS s ON s.site_id = r.site_id \n"+
"WHERE s.project_id = " + dbpool.escape(project_id);
queryHandler(q, callback);
Expand Down
23 changes: 10 additions & 13 deletions app/model/recordings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ var Recordings = {
constraints.push('R.`recording_id` = ' + dbpool.escape(Number(options.recording_id)));
}

if(!urlquery.id) {
if(!urlquery.id && !urlquery.site) {
steps.push(
dbpool.query("(\n" +
" SELECT site_id FROM sites WHERE project_id = ?\n" +
Expand All @@ -353,15 +353,10 @@ var Recordings = {
}).then(function(){
return Q.all(steps);
}).then(function(){
const sql = `SELECT ${group_by.project_part} ${projection} FROM recordings R JOIN sites S ON S.site_id = R.site_id
WHERE (${constraints.join(") AND (")}) ${group_by.clause} ${order_clause} ${limit_clause}`;
return Q.nfcall(queryHandler, {
sql:
"SELECT " + group_by.project_part + projection + " \n" +
"FROM recordings R \n" +
"JOIN sites S ON S.site_id = R.site_id \n" +
"WHERE (" + constraints.join(") AND (") + ")" +
group_by.clause +
order_clause +
limit_clause,
sql,
typeCast: sqlutil.parseUtcDatetime,
}, data);
}).then(function(query_results){
Expand Down Expand Up @@ -990,7 +985,7 @@ var Recordings = {
if(!recording.site_id || !recording.filename)
callback(new Error("Missing fields"));

var q = "SELECT count(*) as count \n"+
var q = "SELECT count(recording_id) as count \n"+
"FROM recordings \n"+
"WHERE site_id = %s \n"+
"AND uri LIKE %s";
Expand Down Expand Up @@ -1141,8 +1136,10 @@ var Recordings = {
return site.site_id;
})
if (siteIds.length) {
constraints.push("r.site_id IN (?)");
data.push(siteIds);
if (!parameters.sites) {
constraints.push("r.site_id IN (?)");
data.push(siteIds);
}
} else {
constraints.push('1 = 2')
}
Expand Down Expand Up @@ -1665,7 +1662,7 @@ var Recordings = {
return this.buildSearchQuery(filters).then(function(builder){
builder.addProjection.apply(builder, [
's.site_id', 's.name as site', 'pis.site_id IS NOT NULL as imported',
'COUNT(*) as count'
'COUNT(recording_id) as count'
]);
delete builder.orderBy;
builder.setGroupBy('s.site_id');
Expand Down
62 changes: 0 additions & 62 deletions assets/app/app/dashboard/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,73 +25,11 @@ <h3 class="panel-title">
</div>
<div class="panel panel-default">
<ul class="list-group">
<li class="list-group-item">
Progress:&nbsp;
<div class="project-progress">
<ul>
<li class="project-progress-level1 full"
tooltip="Create Project">
<i class="fa fa-plus"></i>
</li>

<li class="project-progress-level1"
ng-class="{ full: sites.length }"
tooltip="Add sites">
<i class="fa fa-map-marker"></i>
</li>

<li class="project-progress-level1"
ng-class="{ full: recMins }"
tooltip="Upload recordings">
<i class="fa fa-cloud-upload"></i>
</li>

<!-- <li class="project-progress-level1"
ng-class="{ full: species.length }"
tooltip="Add species">
<i class="fa fa-paw"></i>
</li>
<li class="project-progress-level2"
ng-class="{ full: valsQty }"
tooltip="Validate recordings">
<i class="fa fa-pencil-square-o"></i>
</li> -->

<!-- <li class="project-progress-level2"
ng-class="{ full: trainingSetsQty }"
tooltip="Create training sets">
<i class="fa fa-list-ol"></i>
</li>
<li class="project-progress-level3"
ng-class="{ full: modelsQty }"
tooltip="Create models">
<i class="fa fa-send-o"></i>
</li>
<li class="project-progress-level3"
ng-class="{ full: classiQty }"
tooltip="Run Clasification">
<i class="fa fa-rocket"></i>
</li> -->
</ul>
</div>
</li>
<li class="list-group-item">Sites: {{ sites.length }}</li>
<li class="list-group-item">Recordings: {{ recsQty | number }}</li>
<li class="list-group-item">Templates: {{ templateQty }}</li>
<li class="list-group-item">Pattern Matchings: {{ pmQty }}</li>
<li class="list-group-item">Validated species: <i ng-show="isSpeciesLoading" class="fa fa-spinner fa-spin ng-hide" style="font-size: 18px; position: relative;"></i>{{ speciesQty }}</li>
<!-- <li class="list-group-item">
Species (Validated*: {{ (validatedSpecies === -1 ? "-" : validatedSpecies) + "/" + species.length }}, Templates: {{ templateQty }}):
<ul>
<li ng-repeat="s in species | limitTo: 5">
{{ s.species_name }} ({{ s.songtype_name }}){{ s.validate ? "*" :"" }}
</li>
<p ng-show="species.length > 5">...</p>
</ul>
</li> -->
</ul>
</div>
</div>
Expand Down
64 changes: 3 additions & 61 deletions assets/app/app/dashboard/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ angular.module('a2.app.dashboard',[
});
})
.controller('SummaryCtrl', function($scope, Project, a2Templates, a2GoogleMapsLoader, a2TrainingSets, $timeout, notify, $window, $compile, $templateFetch, a2PatternMatching) {
$scope.loading = 6;
$scope.loading = 5;

var done = function() {
if($scope.loading > 0) --$scope.loading;
Expand All @@ -25,68 +25,15 @@ angular.module('a2.app.dashboard',[
Project.getInfo(function(info){
$scope.project = info;
$scope.isSpeciesLoading = true

Project.getProjectTotalSpecies(info.project_id, function(data) {
$scope.speciesQty = data || 0;
$scope.isSpeciesLoading = false
});

done();
});

Project.getClasses(function(species){
$scope.species = species
done()
// $scope.validatedSpecies = -1

// const promises = species.map(s => {
// return new Promise((resolve) =>
// Project.validationBySpeciesSong(s.species, s.songtype, data => {
// const idx = $scope.species.findIndex(sp => sp.species === s.species && sp.songtype === s.songtype)
// if (idx > -1) {
// $scope.species[idx].validate = data.total > 0
// }
// resolve()
// })
// )
// })
// Promise.all(promises).then(() => {
// $scope.$apply(() => {
// $scope.validatedSpecies = $scope.species.reduce((acc, val) => acc + (val.validate ? 1 : 0),0)
// });
// }).catch((e) => {
// console.log("getClasses error", e)
// })
done();
});

// Project.getModels(function(err, models){
// if(err) {
// notify.serverError();
// }

// $scope.modelsQty = models.length;
// done();
// });

// Project.getClassi(function(err, classi){
// if(err) {
// notify.serverError();
// }

// $scope.classiQty = classi.length;
// done();
// });

// Project.validationsCount(function(count){
// $scope.valsQty = count;
// done();
// });

// a2TrainingSets.getList(function(trainingSets){
// $scope.trainingSetsQty = trainingSets.length;
// done();
// });

Project.getSites(function(sites) {
$scope.sites = sites;
done();
Expand Down Expand Up @@ -123,11 +70,6 @@ angular.module('a2.app.dashboard',[

});

Project.getUsage().success(function(data) {
$scope.recMins = data.min_usage;
done();
});

Project.getRecTotalQty(function(data) {
$scope.recsQty = data;
done();
Expand Down
4 changes: 2 additions & 2 deletions assets/app/home.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ angular.module('a2.home', [
project.patternMatchingsTotal = data.count || 0;
project.isLoading = false;
});
$http.get('/api/project/' + project.url + '/recordings/search-count', {project_id: project.id}).success(function(data) {
project.recCount = data.map((item) => { return item.count }).reduce((a, b) => a + b, 0);
$http.get('/api/project/' + project.url + '/recordings/count', {project_id: project.id}).success(function(data) {
project.recCount = data.count;
project.isLoading = false;
})
$http.get('/api/project/' + project.url + '/recordings/species-count', {project_id: project.id}).success(function(data) {
Expand Down

0 comments on commit 0fbcf49

Please sign in to comment.