From 7ad5b519c945994b38bf7232af2722a23566459a Mon Sep 17 00:00:00 2001 From: Bert Spaan Date: Wed, 8 Jan 2020 15:50:37 +0100 Subject: [PATCH 1/4] version v1.1.0. removed 'dataset' and 'table' from rows, added 'schema' instead. Changed schema base URIs to new location. --- README.md | 8 +- dataset.json | 8 +- meta/auth.json | 4 +- meta/provenance.json | 4 +- meta/relation.json | 6 +- meta/unit.json | 5 +- package.json | 11 +- row-meta-schema.json | 38 ++---- schema.json | 36 +++--- table.json | 6 +- test/datasets/ref-tables.json | 6 +- .../invalid-geometry-type.json | 14 +-- .../invalid-property-name.json | 14 +-- .../row-meta-schemas/missing-description.json | 14 +-- test/row-meta-schemas/missing-id.json | 6 +- test/row-meta-schemas/oneof-root.json | 8 +- test/row-meta-schemas/simple-schema.json | 14 +-- test/row-meta-schemas/without-type.json | 14 +-- test/test.js | 7 +- test/validate.js | 58 +++++++++ validate.js | 119 ------------------ 21 files changed, 146 insertions(+), 254 deletions(-) create mode 100644 test/validate.js delete mode 100644 validate.js diff --git a/README.md b/README.md index 626e240c1..6edf55709 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ GeoJSON You can use any JSON Schema validator to validate data against a JSON Schema. -To validate data from your browser, you can use the [Amsterdam Schema Validator 👩🏼‍🏫](https://observablehq.com/@bertspaan/amsterdam-schema-validator). With the `data=data:text/x-url,` and `schema=data:text/x-url,` URL parameters, you can load data and schema JSON files from URLs. For example, to verify the schema https://static.amsterdam.nl/schemas/dataset/bag/pand with meta-schema https://static.amsterdam.nl/schemas/core/meta/object@v0.0.1, open the following link: +To validate data from your browser, you can use the [Amsterdam Schema Validator 👩🏼‍🏫](https://observablehq.com/@bertspaan/amsterdam-schema-validator). With the `data=data:text/x-url,` and `schema=data:text/x-url,` URL parameters, you can load data and schema JSON files from URLs. For example, to verify the schema https://schemas.data.amsterdam.nl/dataset/bag/pand with meta-schema https://schemas.data.amsterdam.nl/core/meta/object@v0.0.1, open the following link: https://observablehq.com/@bertspaan/amsterdam-schema-validator?data=data:text/x-url,https%3A%2F%2Fams-schema.glitch.me%2Fdataset%2Fbag%2Fpand&schema=data:text/x-url,https%3A%2F%2Fams-schema.glitch.me%2Fcore%2Fmeta%2Fobject%40v0.0.1 @@ -69,9 +69,9 @@ We're currently using [GitHub releases](https://github.com/Amsterdam/amsterdam-s The [Glitch app](https://glitch.com/~ams-schema) mentioned above acts as a proxy, and reads the JSON Schemas from this repository from all available releases: -- https://static.amsterdam.nl/schemas/core/schema ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/schema.json -- https://static.amsterdam.nl/schemas/core/schema@v0.0.1 ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/v0.0.1/schema.json -- https://static.amsterdam.nl/schemas/core/meta/object ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/meta/object.json +- https://schemas.data.amsterdam.nl/core/schema ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/schema.json +- https://schemas.data.amsterdam.nl/core/schema@v0.0.1 ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/v0.0.1/schema.json +- https://schemas.data.amsterdam.nl/core/meta/object ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/meta/object.json ## Examples diff --git a/dataset.json b/dataset.json index 81453a130..c9322e5c4 100644 --- a/dataset.json +++ b/dataset.json @@ -1,10 +1,10 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset@v1.0", + "$id": "https://schemas.data.amsterdam.nl/dataset@v1.1.0", "type": "object", "allOf": [ { - "$ref": "./schema@v1.0#/definitions/basicProperties" + "$ref": "./schema@v1.1.0#/definitions/basicProperties" } ], "required": [ @@ -16,7 +16,7 @@ "const": "dataset" }, "version": { - "$ref": "./schema@v1.0#/definitions/version" + "$ref": "./schema@v1.1.0#/definitions/version" }, "homepage": { "type": "string", @@ -76,7 +76,7 @@ "items": { "oneOf": [ { - "$ref": "./table@v1.0" + "$ref": "./table@v1.1.0" }, { "type": "object", diff --git a/meta/auth.json b/meta/auth.json index b3c782472..3a23da10c 100644 --- a/meta/auth.json +++ b/meta/auth.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/auth@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/auth@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/auth@v1.0": true + "https://schemas.data.amsterdam.nl/meta/auth@v1.1.0": true }, "$recursiveAnchor": true, "title": "Amsterdam Schema authorization", diff --git a/meta/provenance.json b/meta/provenance.json index fffed0acd..1c0aa70a7 100644 --- a/meta/provenance.json +++ b/meta/provenance.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/provenance@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/provenance@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/provenance@v1.0": true + "https://schemas.data.amsterdam.nl/meta/provenance@v1.1.0": true }, "$recursiveAnchor": true, "title": "Amsterdam Schema provenance", diff --git a/meta/relation.json b/meta/relation.json index 308e6523c..9280d118c 100644 --- a/meta/relation.json +++ b/meta/relation.json @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/relation@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/relation@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/relation@v1.0": true + "https://schemas.data.amsterdam.nl/meta/relation@v1.1.0": true }, "$recursiveAnchor": true, "properties": { "relation": { "type": "string", - "format": "uri" + "format": "uri-reference" } } } diff --git a/meta/unit.json b/meta/unit.json index 5a5789ed5..be5496550 100644 --- a/meta/unit.json +++ b/meta/unit.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/unit@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/unit@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/unit@v1.0": true + "https://schemas.data.amsterdam.nl/meta/unit@v1.1.0": true }, "$recursiveAnchor": true, "title": "Amsterdam Schema unit", @@ -13,4 +13,3 @@ } } } - diff --git a/package.json b/package.json index 040fe9c70..4a398ef3f 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,8 @@ { "name": "amsterdam-schema", - "version": "1.0.0", - "description": "", - "main": "cli.js", - "bin": { - "amsterdam-schema": "validate.js" - }, + "version": "1.1.0", + "description": "Amsterdam Schema", "directories": { - "example": "example", "test": "test" }, "scripts": { @@ -17,7 +12,6 @@ "type": "git", "url": "git+https://github.com/Amsterdam/amsterdam-schema.git" }, - "author": "", "license": "ISC", "bugs": { "url": "https://github.com/Amsterdam/amsterdam-schema/issues" @@ -27,7 +21,6 @@ "ajv": "^6.10.2", "axios": "^0.19.0", "glob": "^7.1.4", - "highland": "^2.13.5", "jsonpath": "^1.0.2" } } diff --git a/row-meta-schema.json b/row-meta-schema.json index a6ec5d339..52c0330d2 100644 --- a/row-meta-schema.json +++ b/row-meta-schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/row-meta-schema@v1.0", + "$id": "https://schemas.data.amsterdam.nl/row-meta-schema@v1.1.0", "definitions": { "rootProperty": { "type": "object", @@ -57,23 +57,16 @@ "allOf": [ { "type": "array", - "minItems": 3, + "minItems": 2, "contains": { "const": "id" } }, { "type": "array", - "minItems": 3, + "minItems": 2, "contains": { - "const": "table" - } - }, - { - "type": "array", - "minItems": 3, - "contains": { - "const": "dataset" + "const": "schema" } } ] @@ -82,8 +75,7 @@ "type": "object", "required": [ "id", - "table", - "dataset" + "schema" ], "propertyNames": { "pattern": "^[a-z][A-Za-z0-9]*$" @@ -91,39 +83,31 @@ "properties": { "id": { "type": "object", + "required": ["$ref"], "properties": { "description": { "type": "string" }, "$ref": { - "const": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" - } - } - }, - "table": { - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "$ref": { - "const": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" + "const": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" } } }, - "dataset": { + "schema": { "type": "object", + "required": ["$ref"], "properties": { "description": { "type": "string" }, "$ref": { - "const": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "const": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" } } }, "geometry": { "type": "object", + "required": ["$ref"], "properties": { "$ref": { "type": "string", diff --git a/schema.json b/schema.json index a062dc144..65f728a21 100644 --- a/schema.json +++ b/schema.json @@ -1,13 +1,13 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/schema@v1.0#", + "$id": "https://schemas.data.amsterdam.nl/schema@v1.1.0#", "title": "Amsterdam Schema", "$vocabulary": { "https://json-schema.org/draft/2019-09/vocab/core": true, - "https://static.amsterdam.nl/schemas/meta/auth@v1.0": false, - "https://static.amsterdam.nl/schemas/units@v1.0": false, - "https://static.amsterdam.nl/schemas/relation@v1.0": false, - "https://static.amsterdam.nl/schemas/provenance@v1.0": false + "https://schemas.data.amsterdam.nl/meta/auth@v1.1.0": false, + "https://schemas.data.amsterdam.nl/meta/units@v1.1.0": false, + "https://schemas.data.amsterdam.nl/meta/relation@v1.1.0": false, + "https://schemas.data.amsterdam.nl/meta/provenance@v1.1.0": false }, "$recursiveAnchor": true, "definitions": { @@ -65,22 +65,16 @@ "table" ] }, + "schema": { + "type": "string", + "format": "uri-reference" + }, "title": { "type": "string" }, "description": { "type": "string" }, - "dataset": { - "$ref": "#/definitions/idString" - }, - "class": { - "$ref": "#/definitions/idString" - }, - "uri": { - "type": "string", - "format": "uri" - }, "year": { "type": "integer" }, @@ -93,13 +87,13 @@ { "oneOf": [ { - "$ref": "./dataset@v1.0" + "$ref": "./dataset@v1.1.0" }, { - "$ref": "./table@v1.0" + "$ref": "./table@v1.1.0" }, { - "$ref": "./row-meta-schema@v1.0" + "$ref": "./row-meta-schema@v1.1.0" } ] }, @@ -112,7 +106,7 @@ } }, "then": { - "$ref": "./dataset@v1.0" + "$ref": "./dataset@v1.1.0" } }, { @@ -124,7 +118,7 @@ } }, "then": { - "$ref": "./table@v1.0" + "$ref": "./table@v1.1.0" } }, { @@ -137,7 +131,7 @@ } }, "then": { - "$ref": "./row-meta-schema@v1.0" + "$ref": "./row-meta-schema@v1.1.0" } } ] diff --git a/table.json b/table.json index 84166e51d..e88923da5 100644 --- a/table.json +++ b/table.json @@ -1,10 +1,10 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/table@v1.0", + "$id": "https://schemas.data.amsterdam.nl/table@v1.1.0", "type": "object", "allOf": [ { - "$ref": "./schema@v1.0#/definitions/basicProperties" + "$ref": "./schema@v1.1.0#/definitions/basicProperties" } ], "required": [ @@ -17,7 +17,7 @@ "schema": { "oneOf": [ { - "$ref": "./row-meta-schema@v1.0" + "$ref": "./row-meta-schema@v1.1.0" }, { "type": "object", diff --git a/test/datasets/ref-tables.json b/test/datasets/ref-tables.json index 8483f5182..0a4752201 100644 --- a/test/datasets/ref-tables.json +++ b/test/datasets/ref-tables.json @@ -8,13 +8,13 @@ "language": "nl", "tables": [ { - "$ref": "containers@v1.0" + "$ref": "containers@v1.1.0" }, { - "$ref": "putten@v1.0" + "$ref": "putten@v1.1.0" }, { - "$ref": "inzamellocaties@v1.0" + "$ref": "inzamellocaties@v1.1.0" } ] }, diff --git a/test/row-meta-schemas/invalid-geometry-type.json b/test/row-meta-schemas/invalid-geometry-type.json index f763fedca..43483bc5e 100644 --- a/test/row-meta-schemas/invalid-geometry-type.json +++ b/test/row-meta-schemas/invalid-geometry-type.json @@ -1,23 +1,19 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "additionalProperties": false, "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "geometry": { "$ref": "https://geojson.org/schema/MultiPoint.json", diff --git a/test/row-meta-schemas/invalid-property-name.json b/test/row-meta-schemas/invalid-property-name.json index 6df1ddbfa..4df1a2a11 100644 --- a/test/row-meta-schemas/invalid-property-name.json +++ b/test/row-meta-schemas/invalid-property-name.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "Abc": { "type": "string", diff --git a/test/row-meta-schemas/missing-description.json b/test/row-meta-schemas/missing-description.json index 21710adf2..57c0d4e0c 100644 --- a/test/row-meta-schemas/missing-description.json +++ b/test/row-meta-schemas/missing-description.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "value": { "type": "integer" diff --git a/test/row-meta-schemas/missing-id.json b/test/row-meta-schemas/missing-id.json index 05693b2cd..bc0bb212e 100644 --- a/test/row-meta-schemas/missing-id.json +++ b/test/row-meta-schemas/missing-id.json @@ -1,7 +1,7 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "dataset", @@ -10,10 +10,10 @@ ], "properties": { "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/table" }, "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/dataset" }, "geometry": { "$ref": "https://geojson.org/schema/Geometry.json", diff --git a/test/row-meta-schemas/oneof-root.json b/test/row-meta-schemas/oneof-root.json index 40d5d1d55..dae9f4adb 100644 --- a/test/row-meta-schemas/oneof-root.json +++ b/test/row-meta-schemas/oneof-root.json @@ -1,7 +1,7 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", @@ -10,13 +10,13 @@ ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/table" }, "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/dataset" } }, "oneOf": [ diff --git a/test/row-meta-schemas/simple-schema.json b/test/row-meta-schemas/simple-schema.json index 26aad73cd..f5dc7ce08 100644 --- a/test/row-meta-schemas/simple-schema.json +++ b/test/row-meta-schemas/simple-schema.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "value": { "description": "Waarde van object", diff --git a/test/row-meta-schemas/without-type.json b/test/row-meta-schemas/without-type.json index 88032174d..eb0d576f5 100644 --- a/test/row-meta-schemas/without-type.json +++ b/test/row-meta-schemas/without-type.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "value": { "description": "Invalid property" diff --git a/test/test.js b/test/test.js index b877a1816..168666288 100755 --- a/test/test.js +++ b/test/test.js @@ -7,7 +7,10 @@ const { promisify } = require('util') const glob = promisify(require('glob')) const jsonpath = require('jsonpath') -const validate = require('../validate') +const validate = require('./validate') + +const schema = require('../schema.json') +const schemaUri = schema.$id function readJSONFile (filename) { try { @@ -75,7 +78,7 @@ async function runAllTests () { let results = false try { - const validator = await validate.createValidator('https://static.amsterdam.nl/schemas/schema@v1.0#', schemas) + const validator = await validate.createValidator(schemaUri, schemas) const tests = await readJSONFiles(path.join(__dirname, '*/*.json')) diff --git a/test/validate.js b/test/validate.js new file mode 100644 index 000000000..88beda1a8 --- /dev/null +++ b/test/validate.js @@ -0,0 +1,58 @@ +#!/usr/bin/env node + +const Ajv = require('ajv') +const axios = require('axios') + +function loadSchema (uri) { + return axios.get(uri) + .then((response) => response.data) +} + +function ValidationException (data, errors) { + const error = new Error('Validation exception') + error.name = 'ValidationException' + error.data = data + error.errors = errors + return error +} + +async function createValidatorAsync (schema) { + const ajv = new Ajv({ + loadSchema + }) + + const validate = await ajv.compileAsync(schema) + + return function (data) { + const valid = validate(data) + if (!valid) { + throw new ValidationException(data, validate.errors) + } + + return true + } +} + +async function createValidator (schemaId, schemas) { + const ajv = new Ajv({schemas}) + + const validate = ajv.getSchema(schemaId) + + if (!validate) { + throw new Error(`Can't find schema with ID ${schemaId}`) + } + + return function (data) { + const valid = validate(data) + if (!valid) { + throw new ValidationException(data, validate.errors) + } + + return true + } +} + +module.exports = { + createValidatorAsync, + createValidator +} diff --git a/validate.js b/validate.js deleted file mode 100644 index 59dc2590b..000000000 --- a/validate.js +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env node - -// const fs = require('fs') -// const path = require('path') -const Ajv = require('ajv') -const axios = require('axios') - -// const H = require('highland') -// const argv = require('yargs') -// .option('schemas', { -// alias: 's', -// type: 'array', -// describe: 'JSON Schemas to use (relative paths)' -// }) -// .option('id', { -// alias: 'i', -// type: 'string', -// describe: '$id of schema to use for validation' -// }) -// .demandOption(['id', 'schemas'], 'Please provide both id and schemas arguments') -// .argv - -// const Validate = require('./lib/validate') - -function loadSchema (uri) { - return axios.get(uri) - .then((response) => response.data) -} - -function ValidationException (data, errors) { - const error = new Error('Validation exception') - error.name = 'ValidationException' - error.data = data - error.errors = errors - return error -} - -async function createValidatorAsync (schema) { - const ajv = new Ajv({ - loadSchema - }) - - const validate = await ajv.compileAsync(schema) - - return function (data) { - const valid = validate(data) - if (!valid) { - throw new ValidationException(data, validate.errors) - } - - return true - } -} - -async function createValidator (schemaId, schemas) { - const ajv = new Ajv({schemas}) - - - const validate = ajv.getSchema(schemaId) - - if (!validate) { - throw new Error(`Can't find schema with ID ${schemaId}`) - } - - return function (data) { - const valid = validate(data) - if (!valid) { - throw new ValidationException(data, validate.errors) - } - - return true - } -} - -// if (process.stdin.isTTY) { -// return console.error(`Usage: ${name} [-f jsonPaths] [-o file] FILE\n` + -// ` -f, --flatten JSON array of JSON paths, used to flatten nested data field\n` + -// ` -o, --output Path to output file, if not given, ${name} uses stdout`) - -// const stream = ((argv._.length ? fs.createReadStream(argv._[0], 'utf8') : process.stdin)) - -// const schemas = argv.schemas -// .map((schemaPath) => JSON.parse(fs.readFileSync(path.join('.', schemaPath), 'utf8'))) - -// const validate = Validate.createValidator(argv.id, schemas) - -// H(process.stdin) -// .split() -// .compact() -// .map(JSON.parse) -// .map((object) => validate(object)) -// .errors((err, push) => { -// if (err.name === 'ValidationException') { -// let id -// if (err.data.$id) { -// id = err.data.$id -// } else { -// id = `${err.data.type}:${err.data.id}` -// } - -// console.error(`Errors validating ${id}:`) -// console.error(err.errors) -// } else { -// console.error({ -// name: err.name, -// message: err.message -// }) -// } -// }) -// .compact() -// .map(JSON.stringify) -// .intersperse('\n') -// .pipe(process.stdout) -// } - -module.exports = { - createValidatorAsync, - createValidator -} From 971d9f75b8e71f68da6f5e6d3d452d30f8cceb55 Mon Sep 17 00:00:00 2001 From: Bert Spaan Date: Mon, 25 Nov 2019 14:21:17 +0100 Subject: [PATCH 2/4] added more dcat fields, and fixed some old version numbers --- dataset.json | 20 +++++++++---------- schema.json | 13 +++++++++++- test/datasets/ref-tables.json | 6 +++--- .../invalid-property-name.json | 6 +++--- test/row-meta-schemas/missing-id.json | 4 ++-- test/row-meta-schemas/without-type.json | 6 +++--- validate.js | 5 +++++ 7 files changed, 38 insertions(+), 22 deletions(-) diff --git a/dataset.json b/dataset.json index 608229166..81453a130 100644 --- a/dataset.json +++ b/dataset.json @@ -27,14 +27,6 @@ "minLength": 2, "maxLength": 3 }, - "dateIssued": { - "type": "string", - "format": "date-time" - }, - "dateModified": { - "type": "string", - "format": "date-time" - }, "accrualPeriodicity": { "type": "string" }, @@ -53,14 +45,22 @@ "publisher": { "type": "string" }, + "owner": { + "type": "string" + }, "keywords": { "type": "array", "items": { "type": "string" } }, - "license": { - "type": "string" + "hasBeginning": { + "type": "string", + "format": "date-time" + }, + "hasEnd": { + "type": "string", + "format": "date-time" }, "crs": { "description": "Coordinate reference system", diff --git a/schema.json b/schema.json index 6a7592869..a062dc144 100644 --- a/schema.json +++ b/schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/schema@v0.1#", + "$id": "https://static.amsterdam.nl/schemas/schema@v1.0#", "title": "Amsterdam Schema", "$vocabulary": { "https://json-schema.org/draft/2019-09/vocab/core": true, @@ -29,6 +29,17 @@ }, "description": { "$ref": "#/definitions/description" + }, + "dateCreated": { + "type": "string", + "format": "date-time" + }, + "dateModified": { + "type": "string", + "format": "date-time" + }, + "license": { + "type": "string" } } }, diff --git a/test/datasets/ref-tables.json b/test/datasets/ref-tables.json index 179db0bbb..8483f5182 100644 --- a/test/datasets/ref-tables.json +++ b/test/datasets/ref-tables.json @@ -8,13 +8,13 @@ "language": "nl", "tables": [ { - "$ref": "containers@v0.1" + "$ref": "containers@v1.0" }, { - "$ref": "putten@v0.1" + "$ref": "putten@v1.0" }, { - "$ref": "inzamellocaties@v0.1" + "$ref": "inzamellocaties@v1.0" } ] }, diff --git a/test/row-meta-schemas/invalid-property-name.json b/test/row-meta-schemas/invalid-property-name.json index b74cb4639..6df1ddbfa 100644 --- a/test/row-meta-schemas/invalid-property-name.json +++ b/test/row-meta-schemas/invalid-property-name.json @@ -10,13 +10,13 @@ ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/id" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" }, "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/table" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" }, "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/dataset" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" }, "Abc": { "type": "string", diff --git a/test/row-meta-schemas/missing-id.json b/test/row-meta-schemas/missing-id.json index 77c5d87ca..05693b2cd 100644 --- a/test/row-meta-schemas/missing-id.json +++ b/test/row-meta-schemas/missing-id.json @@ -10,10 +10,10 @@ ], "properties": { "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/table" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" }, "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/dataset" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" }, "geometry": { "$ref": "https://geojson.org/schema/Geometry.json", diff --git a/test/row-meta-schemas/without-type.json b/test/row-meta-schemas/without-type.json index ce4a04387..88032174d 100644 --- a/test/row-meta-schemas/without-type.json +++ b/test/row-meta-schemas/without-type.json @@ -10,13 +10,13 @@ ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/id" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" }, "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/table" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" }, "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v0.1#/definitions/dataset" + "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" }, "value": { "description": "Invalid property" diff --git a/validate.js b/validate.js index 0dda36ff5..59dc2590b 100644 --- a/validate.js +++ b/validate.js @@ -55,8 +55,13 @@ async function createValidatorAsync (schema) { async function createValidator (schemaId, schemas) { const ajv = new Ajv({schemas}) + const validate = ajv.getSchema(schemaId) + if (!validate) { + throw new Error(`Can't find schema with ID ${schemaId}`) + } + return function (data) { const valid = validate(data) if (!valid) { From a1904d872a81bdb694b92f49d45d2e713be4a89d Mon Sep 17 00:00:00 2001 From: Bert Spaan Date: Mon, 25 Nov 2019 15:53:28 +0100 Subject: [PATCH 3/4] updated readme --- README.md | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b2cddc2c3..626e240c1 100644 --- a/README.md +++ b/README.md @@ -4,21 +4,21 @@ # Amsterdam Schema -Een Amsterdam Schema is: - -1. JSON-metadata over dataset, -2. JSON-metadata over één of meer tabellen in deze dataset, -3. En, per tabel, een JSON Schema die de inhoud van en metadata over de velden in deze tabel beschrijft en kan valideren. +This repository contains a work-in-progress version of the Amsterdam Schema. Currently, Amsterdam Schema is a set of [JSON Schemas](https://json-schema.org/) and meta-schemas. The goal of this project is to describe and validate [open data published by the City of Amsterdam](https://api.data.amsterdam.nl/api/). Amsterdam Schema will be used to make the import, storage and publishing layers of our APIs more generic, easier to maintain, and better documented. -In deze Amsterdam Scheme-repository staan: +This repository contains JSON Schemas to make sure every dataset published by the City of Amsterdam always contains the right metadata and is of the right form. -1. JSON Schema om metadata over dataset te valideren, -2. JSON Schema om metadata over tabel te validaren, -3. En een JSON Schema metaschema tabel-JSON Schema's te valideren +An instance of Amsterdam Schema exists of: +1. Metadata about a single dataset; +2. Metadata about each table in this dataset; +3. For each table, a JSON Schema to describe and validate the data in these tables. +This repository contains: -This repository contains a work-in-progress version of the Amsterdam Schema. Currently, Amsterdam Schema is a set of [JSON Schemas](https://json-schema.org/) and meta-schemas. The goal of this project is to describe and validate [open data published by the City of Amsterdam](https://api.data.amsterdam.nl/api/). Amsterdam Schema will be used to make the import, storage and publishing layers of our APIs more generic, easier to maintain, and better documented. +1. A JSON Schema to validate dataset metadata; +2. A JSON Schema to validate table metadata; +3. A JSON Schema _metaschema_ to validate the JSON Schema that describes table data. __Amsterdam Schema is developed by the City of Amsterdam, but the tools and concepts created in this project can be used in any city.__ @@ -30,8 +30,6 @@ For more information, see (some of these pages are in Dutch): - [Amsterdam Schema Validator 👩🏼‍🏫](https://observablehq.com/@bertspaan/amsterdam-schema-validator) - [Amsterdam Schema Tools](https://github.com/Amsterdam/amsterdam-schema-tools) -__All schemas currently use the https://ams-schema.glitch.me/ base URI. This URI will change to something like https://schemas.data.amsterdam.nl/ very soon. Currently, a simple proxy running on Glitch is forwarding requests to GitHub. For details, see https://glitch.com/~ams-schema.__ - ## Schemas Amsterdam Schema aims to restrict the structure and format of data accepted by Amsterdam's open data systems, in order to make the storing and publishing of different datasets more structured, simpler and better documented. @@ -40,12 +38,12 @@ Amsterdam Schema consists of a standard library of available data types, in the ## Concepts -| Type | Description | -|:-----------|:---------------------------------------| -| Dataset | | -| Table | een table, klasse, | -| Row | een row in table, een regel in CSV, etc. | -| Field | in JSON schema property, uitlegfgen +| Type | Description | +|:-----------|:-----------------------------------------| +| Dataset | | +| Table | a table, class, etc. | +| Row | een row in table, een regel in CSV, etc. | +| Field | in JSON schema: property | For example: @@ -61,7 +59,7 @@ GeoJSON You can use any JSON Schema validator to validate data against a JSON Schema. -To validate data from your browser, you can use the [Amsterdam Schema Validator 👩🏼‍🏫](https://observablehq.com/@bertspaan/amsterdam-schema-validator). With the `data=data:text/x-url,` and `schema=data:text/x-url,` URL parameters, you can load data and schema JSON files from URLs. For example, to verify the schema https://ams-schema.glitch.me/dataset/bag/pand with meta-schema https://ams-schema.glitch.me/core/meta/object@v0.0.1, open the following link: +To validate data from your browser, you can use the [Amsterdam Schema Validator 👩🏼‍🏫](https://observablehq.com/@bertspaan/amsterdam-schema-validator). With the `data=data:text/x-url,` and `schema=data:text/x-url,` URL parameters, you can load data and schema JSON files from URLs. For example, to verify the schema https://static.amsterdam.nl/schemas/dataset/bag/pand with meta-schema https://static.amsterdam.nl/schemas/core/meta/object@v0.0.1, open the following link: https://observablehq.com/@bertspaan/amsterdam-schema-validator?data=data:text/x-url,https%3A%2F%2Fams-schema.glitch.me%2Fdataset%2Fbag%2Fpand&schema=data:text/x-url,https%3A%2F%2Fams-schema.glitch.me%2Fcore%2Fmeta%2Fobject%40v0.0.1 @@ -71,9 +69,9 @@ We're currently using [GitHub releases](https://github.com/Amsterdam/amsterdam-s The [Glitch app](https://glitch.com/~ams-schema) mentioned above acts as a proxy, and reads the JSON Schemas from this repository from all available releases: -- https://ams-schema.glitch.me/core/schema ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/schema.json -- https://ams-schema.glitch.me/core/schema@v0.0.1 ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/v0.0.1/schema.json -- https://ams-schema.glitch.me/core/meta/object ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/meta/object.json +- https://static.amsterdam.nl/schemas/core/schema ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/schema.json +- https://static.amsterdam.nl/schemas/core/schema@v0.0.1 ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/v0.0.1/schema.json +- https://static.amsterdam.nl/schemas/core/meta/object ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/meta/object.json ## Examples From 9397e2d702879101cf0091642603c489f5da40e6 Mon Sep 17 00:00:00 2001 From: Bert Spaan Date: Wed, 8 Jan 2020 15:50:37 +0100 Subject: [PATCH 4/4] version v1.1.0. removed 'dataset' and 'table' from rows, added 'schema' instead. Changed schema base URIs to new location. --- README.md | 8 +- dataset.json | 8 +- meta/auth.json | 4 +- meta/provenance.json | 4 +- meta/relation.json | 6 +- meta/unit.json | 5 +- package.json | 11 +- row-meta-schema.json | 38 ++---- schema.json | 36 +++--- table.json | 6 +- test/datasets/ref-tables.json | 6 +- .../invalid-geometry-type.json | 14 +-- .../invalid-property-name.json | 14 +-- .../row-meta-schemas/missing-description.json | 14 +-- test/row-meta-schemas/missing-id.json | 6 +- test/row-meta-schemas/oneof-root.json | 8 +- test/row-meta-schemas/simple-schema.json | 14 +-- test/row-meta-schemas/without-type.json | 14 +-- test/test.js | 7 +- test/validate.js | 58 +++++++++ validate.js | 119 ------------------ 21 files changed, 146 insertions(+), 254 deletions(-) create mode 100644 test/validate.js delete mode 100644 validate.js diff --git a/README.md b/README.md index 626e240c1..6edf55709 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ GeoJSON You can use any JSON Schema validator to validate data against a JSON Schema. -To validate data from your browser, you can use the [Amsterdam Schema Validator 👩🏼‍🏫](https://observablehq.com/@bertspaan/amsterdam-schema-validator). With the `data=data:text/x-url,` and `schema=data:text/x-url,` URL parameters, you can load data and schema JSON files from URLs. For example, to verify the schema https://static.amsterdam.nl/schemas/dataset/bag/pand with meta-schema https://static.amsterdam.nl/schemas/core/meta/object@v0.0.1, open the following link: +To validate data from your browser, you can use the [Amsterdam Schema Validator 👩🏼‍🏫](https://observablehq.com/@bertspaan/amsterdam-schema-validator). With the `data=data:text/x-url,` and `schema=data:text/x-url,` URL parameters, you can load data and schema JSON files from URLs. For example, to verify the schema https://schemas.data.amsterdam.nl/dataset/bag/pand with meta-schema https://schemas.data.amsterdam.nl/core/meta/object@v0.0.1, open the following link: https://observablehq.com/@bertspaan/amsterdam-schema-validator?data=data:text/x-url,https%3A%2F%2Fams-schema.glitch.me%2Fdataset%2Fbag%2Fpand&schema=data:text/x-url,https%3A%2F%2Fams-schema.glitch.me%2Fcore%2Fmeta%2Fobject%40v0.0.1 @@ -69,9 +69,9 @@ We're currently using [GitHub releases](https://github.com/Amsterdam/amsterdam-s The [Glitch app](https://glitch.com/~ams-schema) mentioned above acts as a proxy, and reads the JSON Schemas from this repository from all available releases: -- https://static.amsterdam.nl/schemas/core/schema ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/schema.json -- https://static.amsterdam.nl/schemas/core/schema@v0.0.1 ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/v0.0.1/schema.json -- https://static.amsterdam.nl/schemas/core/meta/object ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/meta/object.json +- https://schemas.data.amsterdam.nl/core/schema ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/schema.json +- https://schemas.data.amsterdam.nl/core/schema@v0.0.1 ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/v0.0.1/schema.json +- https://schemas.data.amsterdam.nl/core/meta/object ⟶ https://raw.githubusercontent.com/Amsterdam/amsterdam-schema/master/meta/object.json ## Examples diff --git a/dataset.json b/dataset.json index 81453a130..c9322e5c4 100644 --- a/dataset.json +++ b/dataset.json @@ -1,10 +1,10 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset@v1.0", + "$id": "https://schemas.data.amsterdam.nl/dataset@v1.1.0", "type": "object", "allOf": [ { - "$ref": "./schema@v1.0#/definitions/basicProperties" + "$ref": "./schema@v1.1.0#/definitions/basicProperties" } ], "required": [ @@ -16,7 +16,7 @@ "const": "dataset" }, "version": { - "$ref": "./schema@v1.0#/definitions/version" + "$ref": "./schema@v1.1.0#/definitions/version" }, "homepage": { "type": "string", @@ -76,7 +76,7 @@ "items": { "oneOf": [ { - "$ref": "./table@v1.0" + "$ref": "./table@v1.1.0" }, { "type": "object", diff --git a/meta/auth.json b/meta/auth.json index b3c782472..3a23da10c 100644 --- a/meta/auth.json +++ b/meta/auth.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/auth@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/auth@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/auth@v1.0": true + "https://schemas.data.amsterdam.nl/meta/auth@v1.1.0": true }, "$recursiveAnchor": true, "title": "Amsterdam Schema authorization", diff --git a/meta/provenance.json b/meta/provenance.json index fffed0acd..1c0aa70a7 100644 --- a/meta/provenance.json +++ b/meta/provenance.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/provenance@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/provenance@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/provenance@v1.0": true + "https://schemas.data.amsterdam.nl/meta/provenance@v1.1.0": true }, "$recursiveAnchor": true, "title": "Amsterdam Schema provenance", diff --git a/meta/relation.json b/meta/relation.json index 308e6523c..9280d118c 100644 --- a/meta/relation.json +++ b/meta/relation.json @@ -1,14 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/relation@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/relation@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/relation@v1.0": true + "https://schemas.data.amsterdam.nl/meta/relation@v1.1.0": true }, "$recursiveAnchor": true, "properties": { "relation": { "type": "string", - "format": "uri" + "format": "uri-reference" } } } diff --git a/meta/unit.json b/meta/unit.json index 5a5789ed5..be5496550 100644 --- a/meta/unit.json +++ b/meta/unit.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/meta/unit@v1.0", + "$id": "https://schemas.data.amsterdam.nl/meta/unit@v1.1.0", "$vocabulary": { - "https://static.amsterdam.nl/schemas/meta/unit@v1.0": true + "https://schemas.data.amsterdam.nl/meta/unit@v1.1.0": true }, "$recursiveAnchor": true, "title": "Amsterdam Schema unit", @@ -13,4 +13,3 @@ } } } - diff --git a/package.json b/package.json index 040fe9c70..4a398ef3f 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,8 @@ { "name": "amsterdam-schema", - "version": "1.0.0", - "description": "", - "main": "cli.js", - "bin": { - "amsterdam-schema": "validate.js" - }, + "version": "1.1.0", + "description": "Amsterdam Schema", "directories": { - "example": "example", "test": "test" }, "scripts": { @@ -17,7 +12,6 @@ "type": "git", "url": "git+https://github.com/Amsterdam/amsterdam-schema.git" }, - "author": "", "license": "ISC", "bugs": { "url": "https://github.com/Amsterdam/amsterdam-schema/issues" @@ -27,7 +21,6 @@ "ajv": "^6.10.2", "axios": "^0.19.0", "glob": "^7.1.4", - "highland": "^2.13.5", "jsonpath": "^1.0.2" } } diff --git a/row-meta-schema.json b/row-meta-schema.json index a6ec5d339..52c0330d2 100644 --- a/row-meta-schema.json +++ b/row-meta-schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/row-meta-schema@v1.0", + "$id": "https://schemas.data.amsterdam.nl/row-meta-schema@v1.1.0", "definitions": { "rootProperty": { "type": "object", @@ -57,23 +57,16 @@ "allOf": [ { "type": "array", - "minItems": 3, + "minItems": 2, "contains": { "const": "id" } }, { "type": "array", - "minItems": 3, + "minItems": 2, "contains": { - "const": "table" - } - }, - { - "type": "array", - "minItems": 3, - "contains": { - "const": "dataset" + "const": "schema" } } ] @@ -82,8 +75,7 @@ "type": "object", "required": [ "id", - "table", - "dataset" + "schema" ], "propertyNames": { "pattern": "^[a-z][A-Za-z0-9]*$" @@ -91,39 +83,31 @@ "properties": { "id": { "type": "object", + "required": ["$ref"], "properties": { "description": { "type": "string" }, "$ref": { - "const": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" - } - } - }, - "table": { - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "$ref": { - "const": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" + "const": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" } } }, - "dataset": { + "schema": { "type": "object", + "required": ["$ref"], "properties": { "description": { "type": "string" }, "$ref": { - "const": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "const": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" } } }, "geometry": { "type": "object", + "required": ["$ref"], "properties": { "$ref": { "type": "string", diff --git a/schema.json b/schema.json index a062dc144..65f728a21 100644 --- a/schema.json +++ b/schema.json @@ -1,13 +1,13 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/schema@v1.0#", + "$id": "https://schemas.data.amsterdam.nl/schema@v1.1.0#", "title": "Amsterdam Schema", "$vocabulary": { "https://json-schema.org/draft/2019-09/vocab/core": true, - "https://static.amsterdam.nl/schemas/meta/auth@v1.0": false, - "https://static.amsterdam.nl/schemas/units@v1.0": false, - "https://static.amsterdam.nl/schemas/relation@v1.0": false, - "https://static.amsterdam.nl/schemas/provenance@v1.0": false + "https://schemas.data.amsterdam.nl/meta/auth@v1.1.0": false, + "https://schemas.data.amsterdam.nl/meta/units@v1.1.0": false, + "https://schemas.data.amsterdam.nl/meta/relation@v1.1.0": false, + "https://schemas.data.amsterdam.nl/meta/provenance@v1.1.0": false }, "$recursiveAnchor": true, "definitions": { @@ -65,22 +65,16 @@ "table" ] }, + "schema": { + "type": "string", + "format": "uri-reference" + }, "title": { "type": "string" }, "description": { "type": "string" }, - "dataset": { - "$ref": "#/definitions/idString" - }, - "class": { - "$ref": "#/definitions/idString" - }, - "uri": { - "type": "string", - "format": "uri" - }, "year": { "type": "integer" }, @@ -93,13 +87,13 @@ { "oneOf": [ { - "$ref": "./dataset@v1.0" + "$ref": "./dataset@v1.1.0" }, { - "$ref": "./table@v1.0" + "$ref": "./table@v1.1.0" }, { - "$ref": "./row-meta-schema@v1.0" + "$ref": "./row-meta-schema@v1.1.0" } ] }, @@ -112,7 +106,7 @@ } }, "then": { - "$ref": "./dataset@v1.0" + "$ref": "./dataset@v1.1.0" } }, { @@ -124,7 +118,7 @@ } }, "then": { - "$ref": "./table@v1.0" + "$ref": "./table@v1.1.0" } }, { @@ -137,7 +131,7 @@ } }, "then": { - "$ref": "./row-meta-schema@v1.0" + "$ref": "./row-meta-schema@v1.1.0" } } ] diff --git a/table.json b/table.json index 84166e51d..e88923da5 100644 --- a/table.json +++ b/table.json @@ -1,10 +1,10 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/table@v1.0", + "$id": "https://schemas.data.amsterdam.nl/table@v1.1.0", "type": "object", "allOf": [ { - "$ref": "./schema@v1.0#/definitions/basicProperties" + "$ref": "./schema@v1.1.0#/definitions/basicProperties" } ], "required": [ @@ -17,7 +17,7 @@ "schema": { "oneOf": [ { - "$ref": "./row-meta-schema@v1.0" + "$ref": "./row-meta-schema@v1.1.0" }, { "type": "object", diff --git a/test/datasets/ref-tables.json b/test/datasets/ref-tables.json index 8483f5182..0a4752201 100644 --- a/test/datasets/ref-tables.json +++ b/test/datasets/ref-tables.json @@ -8,13 +8,13 @@ "language": "nl", "tables": [ { - "$ref": "containers@v1.0" + "$ref": "containers@v1.1.0" }, { - "$ref": "putten@v1.0" + "$ref": "putten@v1.1.0" }, { - "$ref": "inzamellocaties@v1.0" + "$ref": "inzamellocaties@v1.1.0" } ] }, diff --git a/test/row-meta-schemas/invalid-geometry-type.json b/test/row-meta-schemas/invalid-geometry-type.json index f763fedca..43483bc5e 100644 --- a/test/row-meta-schemas/invalid-geometry-type.json +++ b/test/row-meta-schemas/invalid-geometry-type.json @@ -1,23 +1,19 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "additionalProperties": false, "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "geometry": { "$ref": "https://geojson.org/schema/MultiPoint.json", diff --git a/test/row-meta-schemas/invalid-property-name.json b/test/row-meta-schemas/invalid-property-name.json index 6df1ddbfa..4df1a2a11 100644 --- a/test/row-meta-schemas/invalid-property-name.json +++ b/test/row-meta-schemas/invalid-property-name.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "Abc": { "type": "string", diff --git a/test/row-meta-schemas/missing-description.json b/test/row-meta-schemas/missing-description.json index 21710adf2..57c0d4e0c 100644 --- a/test/row-meta-schemas/missing-description.json +++ b/test/row-meta-schemas/missing-description.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "value": { "type": "integer" diff --git a/test/row-meta-schemas/missing-id.json b/test/row-meta-schemas/missing-id.json index 05693b2cd..bc0bb212e 100644 --- a/test/row-meta-schemas/missing-id.json +++ b/test/row-meta-schemas/missing-id.json @@ -1,7 +1,7 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "dataset", @@ -10,10 +10,10 @@ ], "properties": { "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/table" }, "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/dataset" }, "geometry": { "$ref": "https://geojson.org/schema/Geometry.json", diff --git a/test/row-meta-schemas/oneof-root.json b/test/row-meta-schemas/oneof-root.json index 40d5d1d55..dae9f4adb 100644 --- a/test/row-meta-schemas/oneof-root.json +++ b/test/row-meta-schemas/oneof-root.json @@ -1,7 +1,7 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", @@ -10,13 +10,13 @@ ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/table" }, "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/dataset" } }, "oneOf": [ diff --git a/test/row-meta-schemas/simple-schema.json b/test/row-meta-schemas/simple-schema.json index 26aad73cd..f5dc7ce08 100644 --- a/test/row-meta-schemas/simple-schema.json +++ b/test/row-meta-schemas/simple-schema.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "value": { "description": "Waarde van object", diff --git a/test/row-meta-schemas/without-type.json b/test/row-meta-schemas/without-type.json index 88032174d..eb0d576f5 100644 --- a/test/row-meta-schemas/without-type.json +++ b/test/row-meta-schemas/without-type.json @@ -1,22 +1,18 @@ { "data": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://static.amsterdam.nl/schemas/dataset/a/b", + "$id": "https://schemas.data.amsterdam.nl/dataset/a/b", "type": "object", "required": [ "id", - "dataset", - "table" + "schema" ], "properties": { "id": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/id" + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/id" }, - "table": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/table" - }, - "dataset": { - "$ref": "https://static.amsterdam.nl/schemas/schema@v1.0#/definitions/dataset" + "schema": { + "$ref": "https://schemas.data.amsterdam.nl/schema@v1.1.0#/definitions/schema" }, "value": { "description": "Invalid property" diff --git a/test/test.js b/test/test.js index b877a1816..168666288 100755 --- a/test/test.js +++ b/test/test.js @@ -7,7 +7,10 @@ const { promisify } = require('util') const glob = promisify(require('glob')) const jsonpath = require('jsonpath') -const validate = require('../validate') +const validate = require('./validate') + +const schema = require('../schema.json') +const schemaUri = schema.$id function readJSONFile (filename) { try { @@ -75,7 +78,7 @@ async function runAllTests () { let results = false try { - const validator = await validate.createValidator('https://static.amsterdam.nl/schemas/schema@v1.0#', schemas) + const validator = await validate.createValidator(schemaUri, schemas) const tests = await readJSONFiles(path.join(__dirname, '*/*.json')) diff --git a/test/validate.js b/test/validate.js new file mode 100644 index 000000000..88beda1a8 --- /dev/null +++ b/test/validate.js @@ -0,0 +1,58 @@ +#!/usr/bin/env node + +const Ajv = require('ajv') +const axios = require('axios') + +function loadSchema (uri) { + return axios.get(uri) + .then((response) => response.data) +} + +function ValidationException (data, errors) { + const error = new Error('Validation exception') + error.name = 'ValidationException' + error.data = data + error.errors = errors + return error +} + +async function createValidatorAsync (schema) { + const ajv = new Ajv({ + loadSchema + }) + + const validate = await ajv.compileAsync(schema) + + return function (data) { + const valid = validate(data) + if (!valid) { + throw new ValidationException(data, validate.errors) + } + + return true + } +} + +async function createValidator (schemaId, schemas) { + const ajv = new Ajv({schemas}) + + const validate = ajv.getSchema(schemaId) + + if (!validate) { + throw new Error(`Can't find schema with ID ${schemaId}`) + } + + return function (data) { + const valid = validate(data) + if (!valid) { + throw new ValidationException(data, validate.errors) + } + + return true + } +} + +module.exports = { + createValidatorAsync, + createValidator +} diff --git a/validate.js b/validate.js deleted file mode 100644 index 59dc2590b..000000000 --- a/validate.js +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env node - -// const fs = require('fs') -// const path = require('path') -const Ajv = require('ajv') -const axios = require('axios') - -// const H = require('highland') -// const argv = require('yargs') -// .option('schemas', { -// alias: 's', -// type: 'array', -// describe: 'JSON Schemas to use (relative paths)' -// }) -// .option('id', { -// alias: 'i', -// type: 'string', -// describe: '$id of schema to use for validation' -// }) -// .demandOption(['id', 'schemas'], 'Please provide both id and schemas arguments') -// .argv - -// const Validate = require('./lib/validate') - -function loadSchema (uri) { - return axios.get(uri) - .then((response) => response.data) -} - -function ValidationException (data, errors) { - const error = new Error('Validation exception') - error.name = 'ValidationException' - error.data = data - error.errors = errors - return error -} - -async function createValidatorAsync (schema) { - const ajv = new Ajv({ - loadSchema - }) - - const validate = await ajv.compileAsync(schema) - - return function (data) { - const valid = validate(data) - if (!valid) { - throw new ValidationException(data, validate.errors) - } - - return true - } -} - -async function createValidator (schemaId, schemas) { - const ajv = new Ajv({schemas}) - - - const validate = ajv.getSchema(schemaId) - - if (!validate) { - throw new Error(`Can't find schema with ID ${schemaId}`) - } - - return function (data) { - const valid = validate(data) - if (!valid) { - throw new ValidationException(data, validate.errors) - } - - return true - } -} - -// if (process.stdin.isTTY) { -// return console.error(`Usage: ${name} [-f jsonPaths] [-o file] FILE\n` + -// ` -f, --flatten JSON array of JSON paths, used to flatten nested data field\n` + -// ` -o, --output Path to output file, if not given, ${name} uses stdout`) - -// const stream = ((argv._.length ? fs.createReadStream(argv._[0], 'utf8') : process.stdin)) - -// const schemas = argv.schemas -// .map((schemaPath) => JSON.parse(fs.readFileSync(path.join('.', schemaPath), 'utf8'))) - -// const validate = Validate.createValidator(argv.id, schemas) - -// H(process.stdin) -// .split() -// .compact() -// .map(JSON.parse) -// .map((object) => validate(object)) -// .errors((err, push) => { -// if (err.name === 'ValidationException') { -// let id -// if (err.data.$id) { -// id = err.data.$id -// } else { -// id = `${err.data.type}:${err.data.id}` -// } - -// console.error(`Errors validating ${id}:`) -// console.error(err.errors) -// } else { -// console.error({ -// name: err.name, -// message: err.message -// }) -// } -// }) -// .compact() -// .map(JSON.stringify) -// .intersperse('\n') -// .pipe(process.stdout) -// } - -module.exports = { - createValidatorAsync, - createValidator -}