From 696c6b943819983b97d5fcaa35fbe9e6f527d8c7 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 10:14:10 +0200 Subject: [PATCH 01/10] Test for invalid filter field --- api/src/store/search.test.ts | 44 ++++++++++++++++++++++++++++++++++-- api/src/store/search.ts | 6 ++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 34dcf23b..2a60eba2 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -165,13 +165,42 @@ describe('new SearchEngine()', () => { ['solvent', 'Butanol'], ])('filter(\'%s\', \'%s\')', (key: FilterField, value) => { let hits: any; - beforeAll(async () => { + beforeEach(async () => { + client.search.mockClear(); hits = await searchEngine.filter(key, value); }); it('should have called index.search', () => { expect(client.search).toBeCalled(); - // TODO assert arguments + const called = client.search.mock.calls[0][0]; + let expected = { + index: 'podp', + body: { + query: { + match: expect.anything() + } + } + }; + if (key === 'submitter') { + expected = { + index: 'podp', + body: { + query: { + bool: { + should: [ + { + match: expect.anything() + }, + { + match: expect.anything() + } + ], + }, + }, + }, + } as any; + } + expect(called).toEqual(expected); }); it('should return hits', async () => { @@ -179,6 +208,17 @@ describe('new SearchEngine()', () => { expect(hits).toEqual([expected]); }); }); + + describe('filter(invalid field)', () => { + it('should throw Error', async () => { + expect.assertions(1); + try { + await searchEngine.filter('some invalid key' as any, 'somevalue'); + } catch (error) { + expect(error).toEqual(new Error('Invalid filter field')); + } + }); + }); }); describe('with a single metagenome project', () => { diff --git a/api/src/store/search.ts b/api/src/store/search.ts index db4c4fd0..bd82d5a4 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -206,8 +206,12 @@ export class SearchEngine { growth_medium: 'project.experimental.sample_preparation.medium_details.medium_title.keyword', solvent: 'project.experimental.extraction_methods.solvents.solvent_title.keyword', }; + const eskey = key2eskey[key]; + if (!eskey) { + throw Error('Invalid filter field'); + } query.match = {}; - query.match[key2eskey[key]] = value; + query.match[eskey] = value; } const { body } = await this.client.search({ index: this.index, From c45d3ed61996b54c792fbd340cd084ee57d6f38b Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 10:22:46 +0200 Subject: [PATCH 02/10] Added ionization_mode to search filter Refs #132 --- api/src/store/search.test.ts | 2 ++ api/src/store/search.ts | 21 +++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 2a60eba2..98ee40b1 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -163,6 +163,7 @@ describe('new SearchEngine()', () => { ['instrument_type', 'Time-of-flight (TOF)'], ['growth_medium', 'A1 medium'], ['solvent', 'Butanol'], + ['ionization_mode', 'Positive'], ])('filter(\'%s\', \'%s\')', (key: FilterField, value) => { let hits: any; beforeEach(async () => { @@ -294,6 +295,7 @@ async function esGenomeProject() { project.experimental.extraction_methods[1].solvents[0].solvent_title = 'Butanol'; project.experimental.extraction_methods[2].solvents[0].solvent_title = 'Methanol'; project.experimental.instrumentation_methods[0].instrumentation.instrument_title = 'Time-of-flight (TOF)'; + project.experimental.instrumentation_methods[0].mode_title = 'Positive'; const esproject = { _id: 'projectid1', project, diff --git a/api/src/store/search.ts b/api/src/store/search.ts index bd82d5a4..d9686c4e 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -30,10 +30,15 @@ export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, }); const instruments_type_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.instrumentation.properties.instrument.anyOf); + const mode_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.mode.anyOf); doc.project.experimental.instrumentation_methods.forEach((d: any) => { - const title = instruments_type_lookup.get(d.instrumentation.instrument); - if (title) { - d.instrumentation.instrument_title = title; + const instrument_title = instruments_type_lookup.get(d.instrumentation.instrument); + if (instrument_title) { + d.instrumentation.instrument_title = instrument_title; + } + const mode_title = mode_lookup.get(d.mode); + if (mode_title) { + d.mode_title = mode_title; } }); @@ -68,7 +73,10 @@ export function collapseHit(hit: Hit): EnrichedProjectDocument { } ); project.project.experimental.instrumentation_methods.forEach( - (d: any) => delete d.instrumentation.instrument_title + (d: any) => { + delete d.instrumentation.instrument_title; + delete d.mode_title; + } ); project.project.experimental.extraction_methods.forEach( (m: any) => m.solvents.forEach( @@ -91,7 +99,7 @@ export function collapseHit(hit: Hit): EnrichedProjectDocument { return project; } -export type FilterField = 'principal_investigator' | 'submitter' | 'genome_type' | 'species' | 'metagenomic_environment' | 'instrument_type' | 'growth_medium' | 'solvent'; +export type FilterField = 'principal_investigator' | 'submitter' | 'genome_type' | 'species' | 'metagenomic_environment' | 'instrument_type' | 'ionization_mode' | 'growth_medium' | 'solvent'; export class SearchEngine { private schema: any; @@ -203,12 +211,13 @@ export class SearchEngine { species: 'enrichments.genomes.species.scientific_name.keyword', metagenomic_environment: 'project.experimental.sample_preparation.medium_details.metagenomic_environment_title.keyword', instrument_type: 'project.experimental.instrumentation_methods.instrumentation.instrument_title.keyword', + ionization_mode: 'project.experimental.instrumentation_methods.mode_title.keyword', growth_medium: 'project.experimental.sample_preparation.medium_details.medium_title.keyword', solvent: 'project.experimental.extraction_methods.solvents.solvent_title.keyword', }; const eskey = key2eskey[key]; if (!eskey) { - throw Error('Invalid filter field'); + throw new Error('Invalid filter field'); } query.match = {}; query.match[eskey] = value; From 256b1a8daef0fbc242ba449718d3979ee08e5673 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 10:30:05 +0200 Subject: [PATCH 03/10] Added ionization mode to stats api --- api/src/util/stats.test.ts | 9 +++++++++ api/src/util/stats.ts | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/api/src/util/stats.test.ts b/api/src/util/stats.test.ts index bb86b209..20c741df 100644 --- a/api/src/util/stats.test.ts +++ b/api/src/util/stats.test.ts @@ -51,6 +51,9 @@ describe('computeStats()', () => { 'instrument_types': [ ['Time-of-flight (TOF)', 1] ], + 'ionization_modes': [ + ['Positive', 1] + ], 'growth_media': [ ['A1 medium', 1], ['R5 medium', 1], @@ -136,6 +139,9 @@ describe('computeStats()', () => { 'instrument_types': [ ['Time-of-flight (TOF)', 1] ], + 'ionization_modes': [ + ['Positive', 1] + ], 'growth_media': [ ['A1 medium', 1], ['R5 medium', 1], @@ -192,6 +198,9 @@ describe('computeStats()', () => { 'instrument_types': [ ['Time-of-flight (TOF)', 1] ], + 'ionization_modes': [ + ['Positive', 1] + ], 'growth_media': [ ['A1 medium', 1], ['R5 medium', 1], diff --git a/api/src/util/stats.ts b/api/src/util/stats.ts index 71095d22..f49f27fd 100644 --- a/api/src/util/stats.ts +++ b/api/src/util/stats.ts @@ -15,6 +15,7 @@ export interface IStats { species: [string, number][] metagenomic_environment: [string, number][] instrument_types: [string, number][] + ionization_modes: [string, number][] growth_media: [string, number][] solvents: [string, number][] }; @@ -235,6 +236,14 @@ export function computeStats(projects: EnrichedProjectDocument[], schema: any) { instruments_type_lookup, instruments_type_lookup.size ); + const ionization_mode_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.mode.anyOf); + const ionization_modes = countProjectCollectionField( + projects, + (p) => p.project.experimental.instrumentation_methods, + (r) => r.mode, + ionization_mode_lookup, + ionization_mode_lookup.size + ); const growth_media_oneOf = schema.properties.experimental.properties.sample_preparation.items.properties.medium_details.dependencies.medium_type.oneOf[1].properties.medium.anyOf; const growth_media_lookup = enum2map(growth_media_oneOf); @@ -285,6 +294,7 @@ export function computeStats(projects: EnrichedProjectDocument[], schema: any) { submitters: submitters.top, genome_types: genome_types.top, instrument_types: instrument_types.top, + ionization_modes: ionization_modes.top, growth_media: growth_media.top, solvents: solvents.top, species, From f10d4ec05e588cef5b27a2799d2604bfbb9986e0 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 10:42:02 +0200 Subject: [PATCH 04/10] Refresh package-lock.json --- app/package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/package-lock.json b/app/package-lock.json index 6e1f5370..f8d3a057 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,6 +1,6 @@ { "name": "paired-data-platform", - "version": "0.5.0", + "version": "0.6.1", "lockfileVersion": 1, "requires": true, "dependencies": { From 8cc62f6691f11f0cc3f5938c903e206cf9af631d Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 10:46:48 +0200 Subject: [PATCH 05/10] Added ionization modes to stats page Refs #132 --- app/src/api.ts | 1 + app/src/pages/StatsPage.test.tsx | 3 ++- app/src/pages/StatsPage.tsx | 12 ++++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/src/api.ts b/app/src/api.ts index 6d355385..18628e3f 100644 --- a/app/src/api.ts +++ b/app/src/api.ts @@ -43,6 +43,7 @@ export interface IStats { genome_types: [string, number][] species: [string, number][] instrument_types: [string, number][] + ionization_modes: [string, number][] growth_media: [string, number][] metagenomic_environment: [string, number][] solvents: [string, number][] diff --git a/app/src/pages/StatsPage.test.tsx b/app/src/pages/StatsPage.test.tsx index ae6d1337..f3e48e38 100644 --- a/app/src/pages/StatsPage.test.tsx +++ b/app/src/pages/StatsPage.test.tsx @@ -22,6 +22,7 @@ jest.mock('../api', () => ({ genome_types: [['genome', 6]], species: [['Human', 7]], instrument_types: [['Time-of-flight (TOF)', 8]], + ionization_modes: [['Positive', 13]], growth_media: [['A1 medium', 9]], solvents: [['Methanol', 10]], metagenomic_environment: [['Human', 12]] @@ -41,5 +42,5 @@ describe('', () => { it('should render header', () => { expect(wrapper.find('h2')).toHaveLength(1); - }) + }); }) \ No newline at end of file diff --git a/app/src/pages/StatsPage.tsx b/app/src/pages/StatsPage.tsx index da114479..f932dc0a 100644 --- a/app/src/pages/StatsPage.tsx +++ b/app/src/pages/StatsPage.tsx @@ -95,8 +95,16 @@ export const StatsPage = () => { - - + +
+ Ionization modes + + {data.top.ionization_modes.map( + ([value, count]) => + )} + +
+
Growth media From 2043510857da69d8ebeb8642b6cf502b96045515 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 10:55:05 +0200 Subject: [PATCH 06/10] Updated CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01483907..d1241f2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.6.1] 2020-04-16 +### Added + +* Added ionization modes to stats page ([#132](https://github.com/iomega/paired-data-form/issues/132)) + ### Fixed * Enrichments cause Limit of total fields exceeded error in elastic search ([#131](https://github.com/iomega/paired-data-form/issues/131)) From d63b866e442765e9579624610c61feb331ba2fed Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 12:45:20 +0200 Subject: [PATCH 07/10] Added title of ionization_type to es document --- api/src/store/search.test.ts | 1 + api/src/store/search.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 98ee40b1..79026a24 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -296,6 +296,7 @@ async function esGenomeProject() { project.experimental.extraction_methods[2].solvents[0].solvent_title = 'Methanol'; project.experimental.instrumentation_methods[0].instrumentation.instrument_title = 'Time-of-flight (TOF)'; project.experimental.instrumentation_methods[0].mode_title = 'Positive'; + project.experimental.instrumentation_methods[0].ionization_type_title = 'Electrospray Ionization (ESI)'; const esproject = { _id: 'projectid1', project, diff --git a/api/src/store/search.ts b/api/src/store/search.ts index d9686c4e..f4aa5e63 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -31,6 +31,7 @@ export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, const instruments_type_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.instrumentation.properties.instrument.anyOf); const mode_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.mode.anyOf); + const ionization_type_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.ionization.properties.ionization_type.anyOf); doc.project.experimental.instrumentation_methods.forEach((d: any) => { const instrument_title = instruments_type_lookup.get(d.instrumentation.instrument); if (instrument_title) { @@ -40,6 +41,10 @@ export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, if (mode_title) { d.mode_title = mode_title; } + const ionization_type_title = ionization_type_lookup.get(d.ionization.ionization_type); + if (ionization_type_title) { + d.ionization_type_title = ionization_type_title; + } }); const solvents_lookup_enum = schema.properties.experimental.properties.extraction_methods.items.properties.solvents.items.properties.solvent.anyOf; @@ -76,6 +81,7 @@ export function collapseHit(hit: Hit): EnrichedProjectDocument { (d: any) => { delete d.instrumentation.instrument_title; delete d.mode_title; + delete d.ionization_type_title; } ); project.project.experimental.extraction_methods.forEach( From 1ccaca85ce75ab4469d7a1c4639bfc4f84a0ea00 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 12:54:24 +0200 Subject: [PATCH 08/10] Export ionization type as filter field --- api/src/store/search.test.ts | 1 + api/src/store/search.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 79026a24..b83321d6 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -164,6 +164,7 @@ describe('new SearchEngine()', () => { ['growth_medium', 'A1 medium'], ['solvent', 'Butanol'], ['ionization_mode', 'Positive'], + ['ionization_type', 'Electrospray Ionization (ESI)'] ])('filter(\'%s\', \'%s\')', (key: FilterField, value) => { let hits: any; beforeEach(async () => { diff --git a/api/src/store/search.ts b/api/src/store/search.ts index f4aa5e63..079a7d56 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -105,7 +105,7 @@ export function collapseHit(hit: Hit): EnrichedProjectDocument { return project; } -export type FilterField = 'principal_investigator' | 'submitter' | 'genome_type' | 'species' | 'metagenomic_environment' | 'instrument_type' | 'ionization_mode' | 'growth_medium' | 'solvent'; +export type FilterField = 'principal_investigator' | 'submitter' | 'genome_type' | 'species' | 'metagenomic_environment' | 'instrument_type' | 'ionization_mode' | 'ionization_type' | 'growth_medium' | 'solvent'; export class SearchEngine { private schema: any; @@ -218,6 +218,7 @@ export class SearchEngine { metagenomic_environment: 'project.experimental.sample_preparation.medium_details.metagenomic_environment_title.keyword', instrument_type: 'project.experimental.instrumentation_methods.instrumentation.instrument_title.keyword', ionization_mode: 'project.experimental.instrumentation_methods.mode_title.keyword', + ionization_type: 'project.experimental.instrumentation_methods.ionization_type_title.keyword', growth_medium: 'project.experimental.sample_preparation.medium_details.medium_title.keyword', solvent: 'project.experimental.extraction_methods.solvents.solvent_title.keyword', }; From a9ebcc8e3e19f41edb1a3dc78f2a6f6776dc79b6 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 18:17:51 +0200 Subject: [PATCH 09/10] Fix test --- api/src/app.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/app.test.ts b/api/src/app.test.ts index 777a9e53..237aa53a 100644 --- a/api/src/app.test.ts +++ b/api/src/app.test.ts @@ -147,6 +147,7 @@ describe('app', () => { 'submitters': [], 'genome_types': [], 'instrument_types': [], + 'ionization_modes': [], 'growth_media': [], 'solvents': [], 'species': [], From f58f2c612bca5dbcf0cd05410a77eed26fde215c Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Mon, 20 Apr 2020 18:26:16 +0200 Subject: [PATCH 10/10] Moved schema load and schema lookup maps to own utility module --- api/src/app.ts | 4 ++- api/src/controller.ts | 8 ++++-- api/src/migrate.ts | 6 ++--- api/src/store/search.ts | 25 ++++++------------- api/src/util/schema.ts | 49 ++++++++++++++++++++++++++++++++++++ api/src/util/stats.test.ts | 5 ++-- api/src/util/stats.ts | 51 +++++++++++++++----------------------- api/src/validate.ts | 12 +++------ 8 files changed, 94 insertions(+), 66 deletions(-) create mode 100644 api/src/util/schema.ts diff --git a/api/src/app.ts b/api/src/app.ts index 522910c2..63e6286d 100644 --- a/api/src/app.ts +++ b/api/src/app.ts @@ -11,6 +11,7 @@ import { okHandler } from './config/passport'; import { Validator } from './validate'; import { ProjectDocumentStore } from './projectdocumentstore'; import { IOMEGAPairedOmicsDataPlatform } from './schema'; +import { loadSchema } from './util/schema'; export function builder(mystore: ProjectDocumentStore, myenrichqueue: Bull.Queue<[string, IOMEGAPairedOmicsDataPlatform]>) { // Create Express server @@ -19,7 +20,8 @@ export function builder(mystore: ProjectDocumentStore, myenrichqueue: Bull.Queue // Express configuration app.set('port', process.env.PORT || 3000); app.set('store', mystore); - app.set('validator', new Validator()); + app.set('schema', loadSchema()); + app.set('validator', new Validator(app.get('schema'))); app.set('enrichqueue', myenrichqueue); app.use(compression()); app.use(express.json({limit: '1mb'})); diff --git a/api/src/controller.ts b/api/src/controller.ts index 8fcc4ba7..6658f4af 100644 --- a/api/src/controller.ts +++ b/api/src/controller.ts @@ -13,6 +13,10 @@ function getStore(req: Request) { return req.app.get('store') as ProjectDocumentStore; } +function getSchema(req: Request) { + return req.app.get('schema'); +} + function getValidator(req: Request) { return req.app.get('validator') as Validator; } @@ -159,9 +163,9 @@ export function notFoundHandler(error: any, req: Request, res: Response, next: a export async function getStats(req: Request, res: Response) { const store = getStore(req); - const validator = getValidator(req); + const schema = getSchema(req); const projects = await store.listProjects(); - const stats = computeStats(projects, validator.schema); + const stats = computeStats(projects, schema); res.json(stats); } diff --git a/api/src/migrate.ts b/api/src/migrate.ts index fa98c2e9..b9d6f14a 100644 --- a/api/src/migrate.ts +++ b/api/src/migrate.ts @@ -1,9 +1,9 @@ -import { Validator } from './validate'; import { IOMEGAPairedOmicsDataPlatform } from './schema'; import { ProjectDocumentStore } from './projectdocumentstore'; +import { loadSchema } from './util/schema'; -const validator = new Validator(); -const schemaVersion = (validator.schema as any).properties.version.default; +const schema = loadSchema(); +const schemaVersion = schema.properties.version.default; export interface Migration { applicable(p: any): boolean; diff --git a/api/src/store/search.ts b/api/src/store/search.ts index 079a7d56..ab0d04f7 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -1,7 +1,6 @@ import { Client } from '@elastic/elasticsearch'; import { EnrichedProjectDocument } from './enrichments'; -import { loadSchema } from '../validate'; -import { enum2map } from '../util/stats'; +import { loadSchema, Lookups } from '../util/schema'; interface Hit { _id: string; @@ -10,48 +9,40 @@ interface Hit { } export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, schema: any) { + const lookups = new Lookups(schema); const doc = JSON.parse(JSON.stringify(project)); - const growth_media_oneOf = schema.properties.experimental.properties.sample_preparation.items.properties.medium_details.dependencies.medium_type.oneOf[1].properties.medium.anyOf; - const growth_media_lookup = enum2map(growth_media_oneOf); - const metagenomic_environment_oneOf = schema.properties.experimental.properties.sample_preparation.items.properties.medium_details.dependencies.medium_type.oneOf[0].properties.metagenomic_environment.oneOf; - const metagenomic_environment_lookup = enum2map(metagenomic_environment_oneOf); doc.project.experimental.sample_preparation.forEach((d: any) => { if (d.medium_details.medium_type === 'metagenome') { - const metagenomic_environment_title = metagenomic_environment_lookup.get(d.medium_details.metagenomic_environment); + const metagenomic_environment_title = lookups.metagenomic_environment.get(d.medium_details.metagenomic_environment); if (metagenomic_environment_title) { d.medium_details.metagenomic_environment_title = metagenomic_environment_title; } } - const medium_title = growth_media_lookup.get(d.medium_details.medium); + const medium_title = lookups.growth_media.get(d.medium_details.medium); if (medium_title) { d.medium_details.medium_title = medium_title; } }); - const instruments_type_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.instrumentation.properties.instrument.anyOf); - const mode_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.mode.anyOf); - const ionization_type_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.ionization.properties.ionization_type.anyOf); doc.project.experimental.instrumentation_methods.forEach((d: any) => { - const instrument_title = instruments_type_lookup.get(d.instrumentation.instrument); + const instrument_title = lookups.instrument.get(d.instrumentation.instrument); if (instrument_title) { d.instrumentation.instrument_title = instrument_title; } - const mode_title = mode_lookup.get(d.mode); + const mode_title = lookups.ionization_mode.get(d.mode); if (mode_title) { d.mode_title = mode_title; } - const ionization_type_title = ionization_type_lookup.get(d.ionization.ionization_type); + const ionization_type_title = lookups.ionization_type.get(d.ionization.ionization_type); if (ionization_type_title) { d.ionization_type_title = ionization_type_title; } }); - const solvents_lookup_enum = schema.properties.experimental.properties.extraction_methods.items.properties.solvents.items.properties.solvent.anyOf; - const solvents_lookup = enum2map(solvents_lookup_enum); doc.project.experimental.extraction_methods.forEach((m: any) => { m.solvents.forEach((d: any) => { - const title = solvents_lookup.get(d.solvent); + const title = lookups.solvent.get(d.solvent); if (title) { d.solvent_title = title; } diff --git a/api/src/util/schema.ts b/api/src/util/schema.ts new file mode 100644 index 00000000..3f13c0f2 --- /dev/null +++ b/api/src/util/schema.ts @@ -0,0 +1,49 @@ +import fs from 'fs'; +import path from 'path'; + +export const loadSchema = (fn = path.join(__dirname, '../../../app/public/schema.json')) => { + return JSON.parse(fs.readFileSync(fn, 'utf-8')); +}; + +type Lookup = Map; + +function enum2map(choices: any[]): Lookup { + return new Map( + choices.map((c) => [c.enum[0], c.title]) + ); +} + +export class Lookups { + readonly genome_type: Lookup; + readonly growth_media: Lookup; + readonly metagenomic_environment: Lookup; + readonly instrument: Lookup; + readonly ionization_mode: Lookup; + readonly ionization_type: Lookup; + readonly solvent: Lookup; + + constructor(schema = loadSchema()) { + const genome_types_enum: string[] = schema.properties.genomes.items.properties.genome_ID.properties.genome_type.enum; + this.genome_type = new Map( + genome_types_enum.map(s => [s, s]) + ); + this.growth_media = enum2map( + schema.properties.experimental.properties.sample_preparation.items.properties.medium_details.dependencies.medium_type.oneOf[1].properties.medium.anyOf + ); + this.metagenomic_environment = enum2map( + schema.properties.experimental.properties.sample_preparation.items.properties.medium_details.dependencies.medium_type.oneOf[0].properties.metagenomic_environment.oneOf + ); + this.instrument = enum2map( + schema.properties.experimental.properties.instrumentation_methods.items.properties.instrumentation.properties.instrument.anyOf + ); + this.ionization_mode = enum2map( + schema.properties.experimental.properties.instrumentation_methods.items.properties.mode.anyOf + ); + this.ionization_type = enum2map( + schema.properties.experimental.properties.instrumentation_methods.items.properties.ionization.properties.ionization_type.anyOf + ); + this.solvent = enum2map( + schema.properties.experimental.properties.extraction_methods.items.properties.solvents.items.properties.solvent.anyOf + ); + } +} \ No newline at end of file diff --git a/api/src/util/stats.test.ts b/api/src/util/stats.test.ts index 20c741df..d877a742 100644 --- a/api/src/util/stats.test.ts +++ b/api/src/util/stats.test.ts @@ -1,9 +1,9 @@ -import { Validator } from '../validate'; import { loadJSONDocument } from './io'; import { computeStats, IStats } from './stats'; import { IOMEGAPairedOmicsDataPlatform } from '../schema'; import { ProjectEnrichments } from '../enrich'; import { EXAMPLE_PROJECT_JSON_FN } from '../testhelpers'; +import { loadSchema } from './schema'; jest.mock('../projectdocumentstore'); @@ -13,8 +13,7 @@ describe('computeStats()', () => { let schema: object; beforeAll(() => { - const validator = new Validator(); - schema = validator.schema; + schema = loadSchema(); }); describe('with example un-enriched project', () => { diff --git a/api/src/util/stats.ts b/api/src/util/stats.ts index f49f27fd..213f5f33 100644 --- a/api/src/util/stats.ts +++ b/api/src/util/stats.ts @@ -1,5 +1,6 @@ import { EnrichedProjectDocument } from '../store/enrichments'; import { GeneClusterMassSpectraLinks } from '../schema'; +import { Lookups } from './schema'; export interface IStats { global: { @@ -46,7 +47,7 @@ function countProjectCollectionField( projects: EnrichedProjectDocument[], collection_accessor: (project: EnrichedProjectDocument) => any[], field_accessor: (row: any) => string, - lookup: Map, + lookup: ReadonlyMap, top_size = 5, { count_unknowns } = { count_unknowns: true } ) { @@ -82,15 +83,7 @@ function countProjectCollectionField( }; } -export function enum2map(choices: any[]) { - return new Map( - choices.map((c) => [c.enum[0], c.title]) - ); -} - -function countSolvents(projects: EnrichedProjectDocument[], schema: any, top_size = 5) { - const lookup_enum = schema.properties.experimental.properties.extraction_methods.items.properties.solvents.items.properties.solvent.anyOf; - const lookup = enum2map(lookup_enum); +function countSolvents(projects: EnrichedProjectDocument[], lookup: ReadonlyMap, top_size = 5) { const field_counts = new Map(); projects.forEach(project => { const collection = project.project.experimental.extraction_methods; @@ -202,7 +195,11 @@ function countBgcMS2Links(projects: EnrichedProjectDocument[]) { } export function computeStats(projects: EnrichedProjectDocument[], schema: any) { - const principal_investigators = countProjectField(projects, (p) => new Map([[p.project.personal.PI_name ? p.project.personal.PI_name : '', 1]])); + const lookups = new Lookups(schema); + + const principal_investigators = countProjectField(projects, + (p) => new Map([[p.project.personal.PI_name ? p.project.personal.PI_name : '', 1]]) + ); const submitters = countProjectField(projects, (p) => { const submitters_values = new Map(); if (p.project.personal.submitter_name) { @@ -218,37 +215,31 @@ export function computeStats(projects: EnrichedProjectDocument[], schema: any) { return new Map(submitters_values); }); - const genome_types_enum: string[] = schema.properties.genomes.items.properties.genome_ID.properties.genome_type.enum; - const genome_types_lookup = new Map(genome_types_enum.map(s => [s, s])); const genome_types = countProjectCollectionField( projects, (p) => p.project.genomes, (r) => r.genome_ID.genome_type, - genome_types_lookup, - genome_types_lookup.size + lookups.genome_type, + lookups.genome_type.size, ); - const instruments_type_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.instrumentation.properties.instrument.anyOf); const instrument_types = countProjectCollectionField( projects, (p) => p.project.experimental.instrumentation_methods, (r) => r.instrumentation.instrument, - instruments_type_lookup, - instruments_type_lookup.size + lookups.instrument, + lookups.instrument.size ); - const ionization_mode_lookup = enum2map(schema.properties.experimental.properties.instrumentation_methods.items.properties.mode.anyOf); const ionization_modes = countProjectCollectionField( projects, (p) => p.project.experimental.instrumentation_methods, (r) => r.mode, - ionization_mode_lookup, - ionization_mode_lookup.size + lookups.ionization_mode, + lookups.ionization_mode.size ); - const growth_media_oneOf = schema.properties.experimental.properties.sample_preparation.items.properties.medium_details.dependencies.medium_type.oneOf[1].properties.medium.anyOf; - const growth_media_lookup = enum2map(growth_media_oneOf); const metagenome_medium = 'Not available, sample is metagenome'; - growth_media_lookup.set(metagenome_medium, metagenome_medium); + lookups.growth_media.set(metagenome_medium, metagenome_medium); const growth_media = countProjectCollectionField( projects, (p) => p.project.experimental.sample_preparation, @@ -258,11 +249,9 @@ export function computeStats(projects: EnrichedProjectDocument[], schema: any) { } return r.medium_details.medium; }, - growth_media_lookup, - growth_media_lookup.size + lookups.growth_media, + lookups.growth_media.size ); - const metagenomic_environment_oneOf = schema.properties.experimental.properties.sample_preparation.items.properties.medium_details.dependencies.medium_type.oneOf[0].properties.metagenomic_environment.oneOf; - const metagenomic_environment_lookup = enum2map(metagenomic_environment_oneOf); const metagenomic_environment = countProjectCollectionField( projects, (p) => p.project.experimental.sample_preparation, @@ -271,12 +260,12 @@ export function computeStats(projects: EnrichedProjectDocument[], schema: any) { return r.medium_details.metagenomic_environment; } }, - metagenomic_environment_lookup, - metagenomic_environment_lookup.size, + lookups.metagenomic_environment, + lookups.metagenomic_environment.size, { count_unknowns: false } ); - const solvents = countSolvents(projects, schema); + const solvents = countSolvents(projects, lookups.solvent); const metabolome_samples = countMetabolomeSamples(projects); const species = countSpecies(projects); diff --git a/api/src/validate.ts b/api/src/validate.ts index 63f23e7d..aa06c7d2 100644 --- a/api/src/validate.ts +++ b/api/src/validate.ts @@ -1,22 +1,16 @@ import fs from 'fs'; -import path from 'path'; import Ajv from 'ajv'; - -export const loadSchema = (schema_fn = '../../app/public/schema.json') => { - const fn = path.join(__dirname, schema_fn); - return JSON.parse(fs.readFileSync(fn, 'utf-8')); -}; +import { loadSchema } from './util/schema'; export class Validator { ajv: Ajv.Ajv; schema: object; compiled: Ajv.ValidateFunction; - constructor(schema_fn = '../../app/public/schema.json') { // TODO use path relative to this file + constructor(schema = loadSchema()) { this.ajv = new Ajv(); - this.schema = loadSchema(schema_fn); - this.compiled = this.ajv.compile(this.schema); + this.compiled = this.ajv.compile(schema); } validate(data: any) {