diff --git a/.mocharc.json b/.mocharc.json index 40c35fb19..2f955e3b4 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -1,5 +1,5 @@ { "extension": ["ts", "js"], - "spec": "tests/js/**/*.js", + "spec": "tests/js/**/*", "require": "ts-node/register" } diff --git a/compile_ts.mjs b/compile_ts.mjs deleted file mode 100644 index b48b1cc28..000000000 --- a/compile_ts.mjs +++ /dev/null @@ -1,3 +0,0 @@ -import compileTS from "./lib/js/scripts/compileTs.js"; - -compileTS.default("./lib/js/schema", `./lib/js/types.ts`); diff --git a/compile_ts.ts b/compile_ts.ts new file mode 100644 index 000000000..7809a6768 --- /dev/null +++ b/compile_ts.ts @@ -0,0 +1,3 @@ +import compileTS from "./src/js/scripts/compileTs"; + +compileTS("./lib/js/schema", `./lib/js/types.ts`); diff --git a/modules.d.ts b/modules.d.ts new file mode 100644 index 000000000..0e7e3bc7b --- /dev/null +++ b/modules.d.ts @@ -0,0 +1,5 @@ +declare module "json-schema-deref-sync" { + import { JSONSchema7 } from "json-schema"; + + export default function deref(schema: JSONSchema, options: object): JSONSchema7; +} diff --git a/package-lock.json b/package-lock.json index 116281c18..26931c088 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1542,6 +1542,11 @@ "resolved": "https://registry.npmjs.org/@tsconfig/node20/-/node20-20.1.2.tgz", "integrity": "sha512-madaWq2k+LYMEhmcp0fs+OGaLFk0OenpHa4gmI4VEmCKX4PJntQ6fnnGADVFrVkBj0wIdAlQnK/MrlYTHsa1gQ==" }, + "@types/chai": { + "version": "4.3.11", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.11.tgz", + "integrity": "sha512-qQR1dr2rGIHYlJulmr8Ioq3De0Le9E4MJ5AiaeAETJJpndT1uUNHsGFK3L/UIu+rbkQSdj8J/w2bCsBZc/Y5fQ==" + }, "@types/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", @@ -1562,6 +1567,14 @@ "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" }, + "@types/json-schema-merge-allof": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@types/json-schema-merge-allof/-/json-schema-merge-allof-0.6.5.tgz", + "integrity": "sha512-5mS11ZUTyFNUVEMpK3uKoPb6BWL/nLgW/ln2VOiI8OOxKEYC4Gl9O3WjS5P49yqVTfkcbCAPKw3T1O4erUah5g==", + "requires": { + "@types/json-schema": "*" + } + }, "@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -1578,6 +1591,11 @@ "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==" }, + "@types/mocha": { + "version": "10.0.6", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.6.tgz", + "integrity": "sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg==" + }, "@types/node": { "version": "20.11.16", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.16.tgz", @@ -3410,11 +3428,6 @@ "reusify": "^1.0.4" } }, - "file": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/file/-/file-0.2.2.tgz", - "integrity": "sha512-gwabMtChzdnpDJdPEpz8Vr/PX0pU85KailuPV71Zw/un5yJVKvzukhB3qf6O3lnTwIe5CxlMYLh3jOK3w5xrLA==" - }, "file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -4272,6 +4285,11 @@ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "dev": true }, + "json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, "json-schema-compare": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz", diff --git a/package.json b/package.json index a9a666b54..0c74927e6 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,11 @@ "scripts": { "prepublishOnly": "rm -rf lib; npm run transpile", "transpile": "tsc; BUILD_ASSETS=true BUILD_PATH='./lib/js' node build_schemas.js", - "transpile-and-build-assets": "npm run transpile && node compile_ts.mjs", + "transpile-and-build-assets": "npm run transpile && ts-node compile_ts.ts", "build:assets": "node build_schemas.js", "build:js-and-python-modules": "BUILD_PYTHON_MODULES=true node build_schemas.js", "build:assets-with-docs": "BUILD_ASSETS=true node build_schemas.js", - "test": "nyc --reporter=text mocha --bail", + "test": "npm run postinstall && nyc --reporter=text mocha --bail", "lint": "eslint src/js && prettier --write src/js", "lint:fix": "eslint --fix --cache src/js && prettier --write src/js", "prettier": "prettier --check src/js", @@ -70,9 +70,12 @@ "@babel/runtime-corejs3": "7.16.8", "@tsconfig/node14": "^14.1.0", "@tsconfig/node20": "^20.1.2", + "@types/chai": "^4.3.11", + "@types/json-schema-merge-allof": "^0.6.5", + "@types/mocha": "^10.0.6", "ajv": "8.12.0", - "file": "^0.2.2", "js-yaml": "^4.1.0", + "json-schema": "^0.4.0", "json-schema-deref-sync": "0.14.0", "json-schema-merge-allof": "^0.8.1", "json-schema-to-typescript": "^13.1.2", diff --git a/schema/3pse/file/applications/espresso/7.2/pw.x/system.json b/schema/3pse/file/applications/espresso/7.2/pw.x/system.json index 3348626dc..94a288774 100644 --- a/schema/3pse/file/applications/espresso/7.2/pw.x/system.json +++ b/schema/3pse/file/applications/espresso/7.2/pw.x/system.json @@ -503,6 +503,7 @@ "vdw_corr": { "type": "string", "enum": [ + "none", "grimme-d2", "Grimme-D2", "DFT-D", diff --git a/schema/workflow/subworkflow.json b/schema/workflow/subworkflow.json index 4d6316b20..a8d4f6e09 100644 --- a/schema/workflow/subworkflow.json +++ b/schema/workflow/subworkflow.json @@ -13,7 +13,7 @@ "description": "Contains the Units of the subworkflow", "type": "array", "items": { - "$ref": "unit/base.json" + "$ref": "./subworkflow/unit.json" } }, "model": { diff --git a/schema/workflow/subworkflow/unit.json b/schema/workflow/subworkflow/unit.json index 70e6c3b3d..cc9887182 100644 --- a/schema/workflow/subworkflow/unit.json +++ b/schema/workflow/subworkflow/unit.json @@ -1,29 +1,29 @@ { - "$id": "workflow/unit", + "$id": "workflow/subworkflow/unit", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "workflow unit schema", + "title": "workflow subworkflow unit schema", "type": "object", "oneOf": [ { - "$ref": "./unit/io.json" + "$ref": "./../unit/io.json" }, { - "$ref": "./unit/reduce.json" + "$ref": "./../unit/reduce.json" }, { - "$ref": "./unit/condition.json" + "$ref": "./../unit/condition.json" }, { - "$ref": "./unit/assertion.json" + "$ref": "./../unit/assertion.json" }, { - "$ref": "./unit/execution.json" + "$ref": "./../unit/execution.json" }, { - "$ref": "./unit/assignment.json" + "$ref": "./../unit/assignment.json" }, { - "$ref": "./unit/processing.json" + "$ref": "./../unit/processing.json" } ], "discriminator": { diff --git a/src/js/esse/JSONSchemasInterface.ts b/src/js/esse/JSONSchemasInterface.ts index 87617c5df..e483d5e19 100644 --- a/src/js/esse/JSONSchemasInterface.ts +++ b/src/js/esse/JSONSchemasInterface.ts @@ -1,12 +1,27 @@ -import { SchemaObject } from "ajv"; +import Ajv, { SchemaObject } from "ajv"; import fs from "fs"; import path from "path"; import { walkDirSync } from "../scripts/utils"; +import { addAdditionalPropertiesToSchema } from "./schemaUtils"; type Query = { [key in keyof SchemaObject]: { $regex: string } }; -const schemasCache = new Map(); +export interface AnyObject { + [key: string]: unknown; +} + +const ajv = new Ajv({ + removeAdditional: true, + strict: false, + useDefaults: true, + /** + * discriminator fixes default values in oneOf + * @see https://ajv.js.org/guide/modifying-data.html#assigning-defaults + */ + discriminator: true, + coerceTypes: true, // convert "true" => true for boolean or "4" => 4 for integer +}); export function readSchemaFolderSync(folderPath: string) { const schemas: SchemaObject[] = []; @@ -25,6 +40,8 @@ export function readSchemaFolderSync(folderPath: string) { export class JSONSchemasInterface { static schemaFolder = "./lib/js/schema"; + static schemasCache = new Map(); + static setSchemaFolder(schemaFolder: string) { if (this.schemaFolder !== schemaFolder) { this.schemaFolder = schemaFolder; @@ -37,17 +54,17 @@ export class JSONSchemasInterface { schemas.forEach((schema) => { if (schema.$id) { - schemasCache.set(schema.$id, schema); + this.schemasCache.set(schema.$id, schema); } }); } static schemaById(schemaId: string) { - if (schemasCache.size === 0) { + if (this.schemasCache.size === 0) { this.readSchemaFolder(); } - return schemasCache.get(schemaId); + return this.schemasCache.get(schemaId); } /** @@ -71,7 +88,7 @@ export class JSONSchemasInterface { static matchSchema(query: Query) { const searchFields = Object.keys(query) as Array; - return Array.from(schemasCache.values()).find((schema) => { + return Array.from(this.schemasCache.values()).find((schema) => { return searchFields.every((field) => { const queryField = query[field]; const schemaField = schema[field]; @@ -84,4 +101,43 @@ export class JSONSchemasInterface { }); }); } + + private static getAjvValidator(jsonSchema: SchemaObject) { + const schemaKey = jsonSchema.$id as string; + + if (!schemaKey) { + console.log({ + jsonSchema, + }); + } + + let validate = ajv.getSchema(schemaKey); + + if (!validate) { + ajv.addSchema(addAdditionalPropertiesToSchema(jsonSchema), schemaKey); + validate = ajv.getSchema(schemaKey); + } + + if (!validate) { + throw new Error("JSONSchemasInterface AJV validator error"); + } + + return validate; + } + + /** + * Validates a given example against the schema. + * @param example example to validate. + * @param schema schema to validate the example with. + * @returns whether example is valid. + */ + static validate(data: AnyObject, jsonSchema: SchemaObject) { + const validator = this.getAjvValidator(jsonSchema); + const isValid = validator(data); + + return { + isValid, + errors: validator.errors, + }; + } } diff --git a/src/js/esse/index.js b/src/js/esse/index.js deleted file mode 100644 index d85ac78f2..000000000 --- a/src/js/esse/index.js +++ /dev/null @@ -1,69 +0,0 @@ -/* eslint-disable class-methods-use-this */ -import Ajv from "ajv"; -import fs from "fs"; -import yaml from "js-yaml"; -import mergeAllOf from "json-schema-merge-allof"; -import path from "path"; - -import { EXAMPLES_DIR, PROPERTIES_MANIFEST_PATH, SCHEMAS_DIR } from "./settings"; -import { parseIncludeReferenceStatementsByDir } from "./utils"; - -const SCHEMAS = parseIncludeReferenceStatementsByDir(SCHEMAS_DIR); -const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR, true); -const PROPERTIES_MANIFEST = yaml.load( - fs.readFileSync(PROPERTIES_MANIFEST_PATH, { encoding: "utf-8" }), -); -const RESULTS = Object.entries(PROPERTIES_MANIFEST) - .map((k) => (k[1].isResult ? k[0] : null)) - .filter((x) => x); - -export class ESSE { - constructor(config = {}) { - this.schemas = config.schemas || SCHEMAS; - this.wrappedExamples = config.wrappedExamples || EXAMPLES; - this.examples = this.wrappedExamples.map((example) => example.data); - this.propertiesManifest = config.propertiesManifest || PROPERTIES_MANIFEST; - this.results = config.results || RESULTS; - } - - /** - * Validates a given example against the schema. - * @param example {Object|Array} example to validate. - * @param schema {Object} schema to validate the example with. - * @returns {boolean} whether example is valid. - */ - validate = (example, schema) => { - const ajv = new Ajv({ allErrors: true, allowUnionTypes: true, discriminator: true }); - const isValid = ajv.validate(schema, example); - - if (!isValid) { - console.error(ajv.errors); - } - - return isValid; - }; - - writeResolvedSchemas(subfolder, skipMergeAllOff = false) { - this.schemas.forEach((s) => { - let mergedSchema = s; - if (!skipMergeAllOff) { - mergedSchema = mergeAllOf(s, { - resolvers: { defaultResolver: mergeAllOf.options.resolvers.title }, - }); - } - const id_as_path = mergedSchema.$id.replace("-", "_"); - const full_path = `${subfolder}/schema/${id_as_path}.json`; - fs.mkdirSync(path.dirname(full_path), { recursive: true }); - fs.writeFileSync(full_path, JSON.stringify(mergedSchema, null, 4), "utf8"); - }); - } - - writeResolvedExamples(subfolder) { - this.wrappedExamples.forEach((e) => { - const id_as_path = e.path.replace("-", "_"); - const full_path = `${subfolder}/example/${id_as_path}.json`; - fs.mkdirSync(path.dirname(full_path), { recursive: true }); - fs.writeFileSync(full_path, JSON.stringify(e.data, null, 4), "utf8"); - }); - } -} diff --git a/src/js/esse/index.ts b/src/js/esse/index.ts new file mode 100644 index 000000000..ffc063d2f --- /dev/null +++ b/src/js/esse/index.ts @@ -0,0 +1,74 @@ +/* eslint-disable class-methods-use-this */ +import Ajv from "ajv"; +import fs from "fs"; +import yaml from "js-yaml"; +import mergeAllOf from "json-schema-merge-allof"; +import path from "path"; + +import { EXAMPLES_DIR, PROPERTIES_MANIFEST_PATH, SCHEMAS_DIR } from "./settings"; +import { JSONSchema, JSONSchemaWithPath, parseIncludeReferenceStatementsByDir } from "./utils"; + +const SCHEMAS = parseIncludeReferenceStatementsByDir(SCHEMAS_DIR); +const EXAMPLES = parseIncludeReferenceStatementsByDir(EXAMPLES_DIR, true); +const PROPERTIES_MANIFEST = yaml.load( + fs.readFileSync(PROPERTIES_MANIFEST_PATH, { encoding: "utf-8" }), +) as object; +const RESULTS = Object.entries(PROPERTIES_MANIFEST) + .map((k) => (k[1].isResult ? k[0] : null)) + .filter((x) => x) as object; + +interface EsseConfig { + schemas: JSONSchema[]; + examples: JSONSchema[]; + wrappedExamples: JSONSchemaWithPath[]; + propertiesManifest: object; + results: object; +} + +export class ESSE implements EsseConfig { + readonly schemas: JSONSchema[]; + + readonly examples: JSONSchema[]; + + readonly wrappedExamples: JSONSchemaWithPath[]; + + readonly propertiesManifest: object; + + readonly results: object; + + constructor(config?: EsseConfig) { + this.schemas = config?.schemas || SCHEMAS; + this.wrappedExamples = config?.wrappedExamples || EXAMPLES; + this.examples = this.wrappedExamples.map((example) => example.data); + this.propertiesManifest = config?.propertiesManifest || PROPERTIES_MANIFEST; + this.results = config?.results || RESULTS; + } + + writeResolvedSchemas(subfolder: string, skipMergeAllOff = false) { + const schemasFolder = `${subfolder}/schema`; + fs.rmSync(schemasFolder, { recursive: true, force: true }); + this.schemas.forEach((s) => { + let mergedSchema = s; + if (!skipMergeAllOff) { + mergedSchema = mergeAllOf(s, { + resolvers: { defaultResolver: mergeAllOf.options.resolvers.title }, + }); + } + const id_as_path = mergedSchema.$id?.replace(/-/g, "_"); + const full_path = `${schemasFolder}/${id_as_path}.json`; + fs.mkdirSync(path.dirname(full_path), { recursive: true }); + fs.writeFileSync(full_path, JSON.stringify(mergedSchema, null, 4), "utf8"); + }); + } + + writeResolvedExamples(subfolder: string) { + const examplesFolder = `${subfolder}/example`; + fs.rmSync(`${examplesFolder}`, { recursive: true, force: true }); + this.wrappedExamples.forEach((e) => { + const id_as_path = e.path.replace(/-/g, "_"); + const full_path = `${examplesFolder}/${id_as_path}.json`; + fs.mkdirSync(path.dirname(full_path), { recursive: true }); + fs.writeFileSync(full_path, JSON.stringify(e.data, null, 4), "utf8"); + }); + } +} diff --git a/src/js/esse/schemaUtils.ts b/src/js/esse/schemaUtils.ts index d9e48dc5d..267b86a09 100644 --- a/src/js/esse/schemaUtils.ts +++ b/src/js/esse/schemaUtils.ts @@ -1,7 +1,17 @@ -// @ts-nocheck import { SchemaObject } from "ajv"; -export function mapObjectDeep(object: unknown, mapValue: (prop: unknown) => unknown): object { +import { JSONSchema, JSONSchemaDefinition } from "./utils"; + +export type MapSchema = (prop: JSONSchemaDefinition) => JSONSchemaDefinition | undefined; + +export function mapObjectDeep(object: JSONSchemaDefinition, mapValue: MapSchema): JSONSchema; + +export function mapObjectDeep(object: JSONSchemaDefinition[], mapValue: MapSchema): JSONSchema[]; + +export function mapObjectDeep( + object: JSONSchemaDefinition | JSONSchemaDefinition[], + mapValue: MapSchema, +): JSONSchemaDefinition | JSONSchemaDefinition[] { if (typeof object !== "object" || object === null) { return object; } @@ -21,44 +31,21 @@ export function mapObjectDeep(object: unknown, mapValue: (prop: unknown) => unkn return Object.fromEntries(entries); } -export function walkSchema(object: unknown, callback: (prop: unknown) => unknown): object { - if (Array.isArray(object)) { - return object.map((item) => walkSchema(item, callback)); - } - - const cleanObject = callback(object); - - if (typeof cleanObject !== "object" || cleanObject === null) { - return cleanObject; - } - - const entries = Object.entries(cleanObject).map(([key, value]) => { - return [key, walkSchema(value, callback)]; - }); - - return Object.fromEntries(entries); -} - -export function addAdditionalPropertiesToSchema( - schema: SchemaObject, - additionalProperties = false, -) { - // @ts-ignore - return walkSchema(schema, (object) => { - if (typeof object !== "object") { - return object; - } - - const schema = object; +export function addAdditionalPropertiesToSchema(schema: JSONSchema, additionalProperties = false) { + return mapObjectDeep(schema, (object) => { + const schema = object as JSONSchema; - if (schema.type === "object" && schema.properties && !("additionalProperties" in schema)) { + if ( + typeof object === "object" && + schema?.type === "object" && + schema?.properties && + !("additionalProperties" in schema) + ) { return { ...schema, additionalProperties, }; } - - return object; }); } @@ -99,28 +86,15 @@ export function addAdditionalPropertiesToSchema( * } * @returns Clean schema */ -export function cleanSchema(object: T, clean = true): T { - if (Array.isArray(object)) { - return object.map((item) => cleanSchema(item)) as T; - } - - if (typeof object !== "object" || object === null) { - return object; - } - - let cleanObject; - - if ((object as SchemaObject).title && clean) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { title, $schema, ...restObject } = object as SchemaObject; - cleanObject = restObject; - } else { - cleanObject = object; - } - - const entries = Object.entries(cleanObject).map(([key, value]) => { - return [key, cleanSchema(value)]; +export function cleanSchema(schema: JSONSchema) { + let firstRun = true; + + return mapObjectDeep(schema, (object) => { + if (typeof object === "object" && object?.title && firstRun) { + firstRun = false; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { title, $schema, ...restObject } = object as SchemaObject; + return restObject; + } }); - - return Object.fromEntries(entries); } diff --git a/src/js/esse/utils.js b/src/js/esse/utils.js deleted file mode 100644 index 60fdbfbb9..000000000 --- a/src/js/esse/utils.js +++ /dev/null @@ -1,52 +0,0 @@ -import file from "file"; -import deref from "json-schema-deref-sync"; -import path from "path"; - -import { JSONInclude } from "../json_include"; - -/** - * Resolves `include` and `$ref` statements. - * @param filePath {String} file to parse. - */ -export function parseIncludeReferenceStatements(filePath) { - const jsonResolver = new JSONInclude(); - const parsed = jsonResolver.parseIncludeStatements(filePath); - const dirPath = path.dirname(filePath); - let dereferenced = deref(parsed, { baseFolder: dirPath, removeIds: true }); - // handle circular references and use non-dereferenced source - if (dereferenced instanceof Error && dereferenced.message === "Circular self reference") { - dereferenced = parsed; - } - return dereferenced; -} - -/** - * Resolves `include` and `$ref` statements for all the JSON files inside a given directory. - * @param dirPath {String} directory to parse. - */ -export function parseIncludeReferenceStatementsByDir(dirPath, wrapInDataAndPath = false) { - const data = []; - const topDir = path.resolve(__dirname, "../../../"); - file.walkSync(dirPath, (dirPath_, dirs_, files_) => { - files_.forEach((file_) => { - const filePath = path.join(dirPath_, file_); - if (filePath.endsWith(".json")) { - const config = parseIncludeReferenceStatements(filePath); - if (wrapInDataAndPath) { - const _path = path.join( - // remove leading slashes and "example" from path - path - .dirname(filePath) - .replace(path.join(topDir, "example"), "") - .replace(/^\/+/, ""), - path.basename(filePath).replace(".json", ""), - ); - data.push({ data: config, path: _path }); - } else { - data.push(config); - } - } - }); - }); - return data; -} diff --git a/src/js/esse/utils.ts b/src/js/esse/utils.ts new file mode 100644 index 000000000..9d93bc808 --- /dev/null +++ b/src/js/esse/utils.ts @@ -0,0 +1,72 @@ +import { JSONSchema7, JSONSchema7Definition } from "json-schema"; +import deref from "json-schema-deref-sync"; +import path from "path"; + +import { JSONInclude } from "../json_include"; +import { walkDirSync } from "../scripts/utils"; + +export type JSONSchema = JSONSchema7; + +export type JSONSchemaDefinition = JSONSchema7Definition; + +/** + * Resolves `include` and `$ref` statements. + * @param filePath {String} file to parse. + */ +export function parseIncludeReferenceStatements(filePath: string): JSONSchema { + const jsonResolver = new JSONInclude(); + const parsed = jsonResolver.parseIncludeStatements(filePath); + const dirPath = path.dirname(filePath); + let dereferenced = deref(parsed, { baseFolder: dirPath, removeIds: true }); + // handle circular references and use non-dereferenced source + if (dereferenced instanceof Error && dereferenced.message === "Circular self reference") { + dereferenced = parsed as JSONSchema; + } + return dereferenced; +} + +export interface JSONSchemaWithPath { + data: JSONSchema; + path: string; +} + +export function parseIncludeReferenceStatementsByDir( + dirPath: string, + wrapInDataAndPath: true, +): JSONSchemaWithPath[]; + +export function parseIncludeReferenceStatementsByDir( + dirPath: string, + wrapInDataAndPath?: false, +): JSONSchema[]; + +/** + * Resolves `include` and `$ref` statements for all the JSON files inside a given directory. + * @param dirPath directory to parse. + */ +export function parseIncludeReferenceStatementsByDir(dirPath: string, wrapInDataAndPath = false) { + const schemas: JSONSchema[] = []; + const schemasWithPath: JSONSchemaWithPath[] = []; + const topDir = path.resolve(__dirname, "../../../"); + + walkDirSync(dirPath, (filePath) => { + if (filePath.endsWith(".json")) { + const config = parseIncludeReferenceStatements(filePath); + if (wrapInDataAndPath) { + const _path = path.join( + // remove leading slashes and "example" from path + path + .dirname(filePath) + .replace(path.join(topDir, "example"), "") + .replace(/^\/+/, ""), + path.basename(filePath).replace(".json", ""), + ); + schemasWithPath.push({ data: config, path: _path }); + } else { + schemas.push(config); + } + } + }); + + return wrapInDataAndPath ? schemasWithPath : schemas; +} diff --git a/src/js/json_include/index.js b/src/js/json_include/index.ts similarity index 67% rename from src/js/json_include/index.js rename to src/js/json_include/index.ts index 8bbc3e01b..a5a5398bc 100644 --- a/src/js/json_include/index.js +++ b/src/js/json_include/index.ts @@ -4,28 +4,37 @@ import path from "path"; import { INCLUDE_KEY, INCLUDE_VALUE_REGEX, OBJECT_ONLY } from "./settings"; import { isInstanceOf, safeParseJSON } from "./utils"; +interface AnyObject { + [key: string]: AnyObject | AnyObject[]; +} + export class JSONInclude { - constructor() { - this.JSON_INCLUDE_CACHE = {}; - } + JSON_INCLUDE_CACHE: AnyObject = {}; /** * Extracts file name(path) from the include statement. - * @param value {String} string to extract the file name from. + * @param value string to extract the file name from. */ - _getIncludeFileName = (value) => { - if (isInstanceOf(value, "String") && value.search(INCLUDE_VALUE_REGEX) !== -1) { - return value.match(INCLUDE_VALUE_REGEX)[1]; + _getIncludeFileName = (value: unknown) => { + if (typeof value === "string" && value.search(INCLUDE_VALUE_REGEX) !== -1) { + const matched = value.match(INCLUDE_VALUE_REGEX); + return matched && matched[1]; } }; /** * Walks a nested object to resolve include statements. - * @param obj {Object} Object to traverse. - * @param dirpath {String} directory from which `obj` is obtained. Include statements are relative to `obj` path. + * @param obj Object to traverse. + * @param dirpath directory from which `obj` is obtained. Include statements are relative to `obj` path. */ - _walkObjectToInclude(obj, dirpath) { - if (isInstanceOf(obj, "Object")) { + _walkObjectToInclude(obj: AnyObject | AnyObject[], dirpath: string) { + if (isInstanceOf(obj, "Array")) { + for (let i = 0; i < obj.length; i++) { + if (isInstanceOf(obj[i], "Object") || isInstanceOf(obj[i], "Array")) { + this._walkObjectToInclude(obj[i], dirpath); + } + } + } else if (isInstanceOf(obj, "Object")) { if (INCLUDE_KEY in obj) { const includeName = this._getIncludeFileName(obj[INCLUDE_KEY]); if (includeName) { @@ -36,6 +45,7 @@ export class JSONInclude { this.parseIncludeStatements(filePath); } Object.keys(this.JSON_INCLUDE_CACHE[includeName]).forEach((attr) => { + // @ts-ignore obj[attr] = this.JSON_INCLUDE_CACHE[includeName][attr]; }); } @@ -45,21 +55,15 @@ export class JSONInclude { this._walkObjectToInclude(obj[key], dirpath); } }); - } else if (isInstanceOf(obj, "Array")) { - for (let i = 0; i < obj.length; i++) { - if (isInstanceOf(obj[i], "Object") || isInstanceOf(obj[i], "Array")) { - this._walkObjectToInclude(obj[i], dirpath); - } - } } } /** * Resolves `include` statements. - * @param filePath {String} file to parse. + * @param filePath file to parse. */ - parseIncludeStatements(filePath) { - const data = safeParseJSON(fs.readFileSync(filePath, "utf8")); + parseIncludeStatements(filePath: string) { + const data: AnyObject[] = safeParseJSON(fs.readFileSync(filePath, "utf8")); if (OBJECT_ONLY && !isInstanceOf(data, "Object")) { throw new Error( "The JSON file being included should always be a dict rather than a list", diff --git a/src/js/json_include/utils.ts b/src/js/json_include/utils.ts index 946a7c4a6..198a2b2b7 100644 --- a/src/js/json_include/utils.ts +++ b/src/js/json_include/utils.ts @@ -1,4 +1,8 @@ -export function isInstanceOf(object: object, type: string) { +export function isInstanceOf(object: object, type: "Array"): object is object[]; + +export function isInstanceOf(object: object, type: "Object"): object is object; + +export function isInstanceOf(object: object, type: string): boolean { return Object.prototype.toString.call(object).slice(8, -1) === type; } diff --git a/src/js/scripts/compileTs.ts b/src/js/scripts/compileTs.ts index 08265b00c..043cfdedf 100644 --- a/src/js/scripts/compileTs.ts +++ b/src/js/scripts/compileTs.ts @@ -1,5 +1,5 @@ import fs from "fs"; -import { compile } from "json-schema-to-typescript"; +import { compile, JSONSchema } from "json-schema-to-typescript"; import { cleanSchema } from "../esse/schemaUtils"; import { walkDir } from "./utils"; @@ -13,11 +13,11 @@ export default async function compileTS(schemaPath: string, savePath: string) { await walkDir(schemaPath, async (filePath) => { const data = await fs.promises.readFile(filePath, "utf8"); - const schema = cleanSchema(JSON.parse(data), false); + const schema = cleanSchema(JSON.parse(data)); console.log(filePath); - const compiledSchema = await compile(schema, schema.title, { + const compiledSchema = await compile(schema as JSONSchema, schema.title || "", { unreachableDefinitions: true, additionalProperties: false, bannerComment: `/** Schema ${filePath} */`, diff --git a/tests/js/JSONSchemasInterface.tests.js b/tests/js/JSONSchemasInterface.tests.ts similarity index 90% rename from tests/js/JSONSchemasInterface.tests.js rename to tests/js/JSONSchemasInterface.tests.ts index 517dec83c..773a2a22b 100644 --- a/tests/js/JSONSchemasInterface.tests.js +++ b/tests/js/JSONSchemasInterface.tests.ts @@ -4,11 +4,11 @@ import path from "path"; import { JSONSchemasInterface } from "../../src/js/esse/JSONSchemasInterface"; -function assertObject(prop) { +function assertObject(prop: unknown) { expect(prop).to.be.an("object"); } -function assertArray(prop) { +function assertArray(prop: unknown) { expect(prop).to.be.an("array"); } @@ -23,7 +23,9 @@ describe("JSONSchemasInterface", () => { expect(schema?.properties?.inSet).to.be.an("object"); if ( + // @ts-ignore assertObject(schema?.properties?.inSet) && + // @ts-ignore assertObject(schema?.properties?.inSet?.items) ) { expect(schema?.properties?.inSet?.items?.$id).to.be.an("undefined"); @@ -33,7 +35,9 @@ describe("JSONSchemasInterface", () => { expect(schema?.properties?.valueMapFunction?.enum).to.be.an("array"); if ( + // @ts-ignore assertObject(schema?.properties?.valueMapFunction) && + // @ts-ignore assertArray(schema?.properties?.valueMapFunction?.enum) ) { expect(schema?.properties?.valueMapFunction?.enum[0]).to.be.an("string"); diff --git a/tests/js/validate.js b/tests/js/validate.js deleted file mode 100644 index 4b5439640..000000000 --- a/tests/js/validate.js +++ /dev/null @@ -1,42 +0,0 @@ -/* eslint-disable no-unused-expressions */ -// eslint-disable-next-line import/no-extraneous-dependencies -import { expect } from "chai"; -import file from "file"; -import groupBy from "lodash/groupBy"; -import path from "path"; - -import { ESSE } from "../../src/js/esse"; -import { EXAMPLES_DIR, SCHEMAS_DIR } from "../../src/js/esse/settings"; -import { parseIncludeReferenceStatements } from "../../src/js/esse/utils"; - -const esse = new ESSE(); - -describe("validate all examples", () => { - file.walkSync(EXAMPLES_DIR, (dirPath_, dirs_, files_) => { - files_.forEach((file_) => { - const examplePath = path.join(dirPath_, file_); - const schemaPath = examplePath.replace(EXAMPLES_DIR, SCHEMAS_DIR); - if (!examplePath.endsWith(".json") || !schemaPath.endsWith(".json")) { - // ignore files like .DS_Store - return; - } - it(`${examplePath.replace(`${EXAMPLES_DIR}/`, "")}`, () => { - const example = parseIncludeReferenceStatements(examplePath); - const schema = parseIncludeReferenceStatements(schemaPath); - const valid = esse.validate(example, schema); - // eslint-disable-next-line no-unused-expressions - expect(valid).to.be.ok; - }); - }); - }); -}); - -describe("schema titles must be unique or empty", () => { - const repeatedSchemaTitles = Object.entries(groupBy(esse.schemas, "title")) - .filter(([title, groupedValues]) => title !== "undefined" && groupedValues.length > 1) - .map(([title, groupedValues]) => [title, groupedValues.map(({ $id }) => $id)]); - - console.log(repeatedSchemaTitles); - - expect(repeatedSchemaTitles).to.be.an("array").that.is.empty; -}); diff --git a/tests/js/validate.ts b/tests/js/validate.ts new file mode 100644 index 000000000..708380baa --- /dev/null +++ b/tests/js/validate.ts @@ -0,0 +1,52 @@ +/* eslint-disable no-unused-expressions */ +// eslint-disable-next-line import/no-extraneous-dependencies +import { expect } from "chai"; +import fs from "fs"; +import groupBy from "lodash/groupBy"; +import path from "path"; + +import { JSONSchemasInterface } from "../../src/js/esse/JSONSchemasInterface"; +import { walkDirSync } from "../../src/js/scripts/utils"; + +const examplesPath = path.resolve("./lib/js/example"); +const schemasPath = path.resolve("./lib/js/schema"); + +describe("validate all examples", () => { + walkDirSync(examplesPath, (examplePath) => { + if (examplePath.endsWith(".json")) { + const schemaPath = examplePath.replace("/example/", "/schema/"); + const example = JSON.parse(fs.readFileSync(examplePath).toString()); + const schema = JSON.parse(fs.readFileSync(schemaPath).toString()); + + console.log({ + example, + schema, + }); + + const result = JSONSchemasInterface.validate(example, schema); + + if (!result.isValid) { + console.log({ + examplePath, + schemaPath, + errors: JSON.stringify(result.errors), + }); + } + + expect(result.isValid).to.be.true; + } + }); +}); + +describe("schema titles must be unique or empty", () => { + JSONSchemasInterface.setSchemaFolder(schemasPath); + const schemas = JSONSchemasInterface.schemasCache.values(); + const repeatedSchemaTitles = Object.entries(groupBy(schemas, "title")) + .filter(([title, groupedValues]) => title !== "undefined" && groupedValues.length > 1) + // @ts-ignore + .map(([title, groupedValues]) => [title, groupedValues.map(({ $id }) => $id)]); + + console.log(repeatedSchemaTitles); + + expect(repeatedSchemaTitles).to.be.an("array").that.is.empty; +}); diff --git a/tsconfig.json b/tsconfig.json index ba9c73b65..5d0ef355b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,13 +14,13 @@ "forceConsistentCasingInFileNames": true, "strict": true, "skipLibCheck": true, - "types": ["node"], + "types": ["mocha", "node"], "rootDir": "./src", "baseUrl": "./", - "resolveJsonModule": true }, - "include": [ - "./src" + "include": [ + "./src", + "modules.d.ts" ], "exclude": [ "node_modules",