diff --git a/.changeset/bright-hairs-serve.md b/.changeset/bright-hairs-serve.md new file mode 100644 index 000000000..a1eccabfe --- /dev/null +++ b/.changeset/bright-hairs-serve.md @@ -0,0 +1,5 @@ +--- +"@cube-creator/ui": patch +--- + +Increase the default width of metadata panel diff --git a/.changeset/fresh-olives-refuse.md b/.changeset/fresh-olives-refuse.md new file mode 100644 index 000000000..31badc796 --- /dev/null +++ b/.changeset/fresh-olives-refuse.md @@ -0,0 +1,5 @@ +--- +"@cube-creator/ui": patch +--- + +Improve rendering of translated form fields to fill available space diff --git a/.changeset/ten-clocks-shop.md b/.changeset/ten-clocks-shop.md new file mode 100644 index 000000000..5e8b31c46 --- /dev/null +++ b/.changeset/ten-clocks-shop.md @@ -0,0 +1,5 @@ +--- +"@cube-creator/core-api": minor +--- + +Added resources to Opendata.swiss cube metadata (`dcat:Distribution`). re https://gitlab.ldbar.ch/bafu/visualize/-/issues/560 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b0bd700b1..8f275b5cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,15 +7,21 @@ on: jobs: unit-test: runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + command: + - test + - test:queries + - test:cli:rest timeout-minutes: 45 steps: - uses: actions/checkout@v3 - uses: ./.github/workflows/setup-env - - run: yarn test - - run: yarn test:queries - - run: yarn test:cli:rest + - run: yarn ${{ matrix.command }} env: AUTH_RUNNER_CLIENT_SECRET: ${{ secrets.AUTH_RUNNER_CLIENT_SECRET }} + NODE_TLS_REJECT_UNAUTHORIZED: 0 - name: logs on fail if: ${{ failure() }} run: lando logs -s core @@ -47,7 +53,7 @@ jobs: - run: yarn seed-data - name: Hydra e2e tests - run: docker-compose -f docker-compose.yml -f docker-compose.posix.yml run e2e-tests + run: docker compose -f docker-compose.yml -f docker-compose.posix.yml run e2e-tests - name: core logs on fail if: ${{ failure() }} run: lando logs -s core @@ -86,6 +92,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/workflows/setup-env + - run: echo insecure >> ~/.curlrc - run: yarn ${{ matrix.command }} env: AUTH_RUNNER_CLIENT_SECRET: ${{ secrets.AUTH_RUNNER_CLIENT_SECRET }} diff --git a/.local.env b/.local.env index ffbae4f53..a41826917 100644 --- a/.local.env +++ b/.local.env @@ -5,11 +5,11 @@ API_CORE_BASE=https://cube-creator.lndo.site/ UI_BASE=https://app.cube-creator.lndo.site/ # SPARQL endpoints -STORE_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/query -STORE_UPDATE_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/update -STORE_GRAPH_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/data +STORE_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/query +STORE_UPDATE_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/update +STORE_GRAPH_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/data STORE_ENGINE=fuseki -PUBLIC_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions +PUBLIC_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions TRIFID_UI=https://lindas.admin.ch/sparql VISUALIZE_UI=https://visualize.admin.ch @@ -43,13 +43,15 @@ OTEL_LOG_LEVEL=debug MANAGED_DIMENSIONS_GRAPH=https://lindas.admin.ch/cube/dimension MANAGED_DIMENSIONS_API_BASE=https://cube-creator.lndo.site/ MANAGED_DIMENSIONS_BASE=https://ld.admin.ch/cube/ -MANAGED_DIMENSIONS_STORE_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions/query -MANAGED_DIMENSIONS_STORE_UPDATE_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions/update -MANAGED_DIMENSIONS_STORE_GRAPH_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions/data +MANAGED_DIMENSIONS_STORE_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions/query +MANAGED_DIMENSIONS_STORE_UPDATE_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions/update +MANAGED_DIMENSIONS_STORE_GRAPH_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions/data MANAGED_DIMENSIONS_STORE_ENGINE=fuseki # PX Cube sample PX_CUBE_BASE=https://environment.ld.admin.ch/foen/example/px-cube/ PX_CUBE_GRAPH=http://example.org/px-cube -PX_CUBE_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/px-cube/query -PX_CUBE_GRAPH_ENDPOINT=http://db.cube-creator.lndo.site/px-cube/data +PX_CUBE_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/px-cube/query +PX_CUBE_GRAPH_ENDPOINT=https://db.cube-creator.lndo.site/px-cube/data + +NODE_TLS_REJECT_UNAUTHORIZED=0 diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..3c032078a --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +18 diff --git a/README.md b/README.md index 03af437d8..858a23923 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ There are two types of e2e tests: ### API e2e tests -Running the E2E tests can be done using: `docker-compose run --rm e2e-tests`, and `docker-compose run --rm e2e-tests -- --grep pattern` lets you select which tests to run. +Running the E2E tests can be done using: `docker compose run --rm e2e-tests`, and `docker compose run --rm e2e-tests -- --grep pattern` lets you select which tests to run. For brevity, use npm script `npm run test:e2e --grep pattern` diff --git a/apis/core/bootstrap/shapes/dataset.ts b/apis/core/bootstrap/shapes/dataset.ts index c0ad13f99..f1de83575 100644 --- a/apis/core/bootstrap/shapes/dataset.ts +++ b/apis/core/bootstrap/shapes/dataset.ts @@ -5,14 +5,19 @@ import { sparql, turtle } from '@tpluscode/rdf-string' import $rdf from 'rdf-ext' import { Draft } from '@cube-creator/model/Cube' import env from '@cube-creator/core/env' -import { lindasQuery } from '../lib/query' +import namespace from '@rdfjs/namespace' +import { lindasQuery, lindasQueryTemplate } from '../lib/query' + +const euvoc = namespace('http://publications.europa.eu/ontology/euvoc#') const shapeId = shape('dataset/edit-metadata') const temporalFromTo = $rdf.namedNode(shapeId.value + '#temporalFromTo') const vcardOrganization = $rdf.namedNode(shapeId.value + '#vcardOrganization') +const opendataSwissResources = $rdf.namedNode(shapeId.value + '#opendataSwissResources') const mainGroup = $rdf.blankNode() const opendataGroup = $rdf.blankNode() +const opendataResourcesGroup = $rdf.blankNode() const aboutGroup = $rdf.blankNode() const themesQuery = sparql`construct { @@ -46,6 +51,40 @@ const euThemesQuery = sparql`construct { } }` +const fileTypesQuery = sparql`construct { + ?c a ${hydra.Collection} . + ?c ${hydra.member} ?term . + ?term ${rdfs.label} ?name . +} WHERE { + BIND ( bnode('collection') as ?c ) + + graph { + ?term a ${euvoc.FileType} ; + ${skos.prefLabel} ?name ; + ${skos.inScheme} . + + FILTER(regex(?name, "_q_", "i")) + } +}` + +const mediaTypesQuery = sparql`construct { + ?c a ${hydra.Collection} . + ?c ${hydra.member} ?term . + ?term ${rdfs.label} ?name . +} WHERE { + BIND ( bnode('collection') as ?c ) + + graph { + ?term a ${dcterms.MediaType} ; + ${dcterms.identifier} ?imt ; + ${skos.inScheme} . + + BIND(str(?imt) as ?name) + + FILTER(regex(?name, "_q_", "i")) + } +}` + const aboutQuery = sparql`construct { ?c a ${hydra.Collection} . ?c ${hydra.member} ?about . @@ -118,7 +157,8 @@ export const DatasetShape = turtle` ${shapeId} { ${mainGroup} ${rdfs.label} "Cube Metadata" ; ${sh.order} 1 . ${opendataGroup} ${rdfs.label} "Opendata.swiss" ; ${sh.order} 2 . - ${aboutGroup} ${rdfs.label} "About" ; ${sh.order} 3 . + ${opendataResourcesGroup} ${rdfs.label} "Opendata.swiss Resources" ; ${sh.order} 3 . + ${aboutGroup} ${rdfs.label} "About" ; ${sh.order} 4 . ${shapeId} a ${sh.NodeShape}, ${hydra.Resource} ; ${sh.targetClass} ${_void.Dataset} ; @@ -317,7 +357,7 @@ ${shapeId} { ${sh.order} 105 ; ${sh.description} "The Data Theme Category for the classification in the European Data Catalogs."; ${sh.nodeKind} ${sh.IRI} ; - ${sh.class} ; + ${sh.class} ${euvoc.DataTheme} ; ${hydra.collection} ${lindasQuery(euThemesQuery)} ; ${sh.group} ${aboutGroup} ; ] ; @@ -384,6 +424,83 @@ ${shapeId} { ${sh.message} "Metadata must not have the dcterms:identifier property. Please remove it in advanced RDF editor" ; ${sh.group} ${mainGroup} ; ] ; + ${sh.property} [ + ${sh.name} "Resources" ; + ${sh.path} ${dcat.distribution} ; + ${sh.class} ${dcat.Distribution} ; + ${sh.nodeKind} ${sh.BlankNode} ; + ${dash.editor} ${dash.DetailsEditor} ; + ${sh.node} ${opendataSwissResources} ; + ${sh.order} 10 ; + ${sh.group} ${opendataResourcesGroup} ; + ] ; + . + + ${opendataSwissResources} a ${sh.NodeShape} ; + ${sh.property} [ + ${sh.name} "Access URL" ; + ${sh.path} ${dcat.accessURL} ; + ${sh.minCount} 1 ; + ${sh.nodeKind} ${sh.IRI} ; + ${sh.order} 10 ; + ${sh.message} "Web page address to access the data is required" ; + ] , [ + ${sh.name} "Download URL" ; + ${sh.path} ${dcat.downloadURL} ; + ${sh.nodeKind} ${sh.IRI} ; + ${sh.order} 20 ; + ] , [ + ${sh.name} "Title" ; + ${sh.path} ${dcterms.title} ; + ${sh.maxCount} 1 ; + ${sh.order} 30 ; + ${sh.datatype} ${rdf.langString} ; + ${sh.languageIn} ( ${supportedLanguages} ) ; + ] , [ + ${sh.name} "Description" ; + ${sh.path} ${dcterms.description} ; + ${sh.order} 40 ; + ${sh.datatype} ${rdf.langString} ; + ${sh.languageIn} ( ${supportedLanguages} ) ; + ] , [ + ${sh.name} "Media type" ; + ${sh.path} ${dcat.mediaType} ; + ${sh.maxCount} 1 ; + ${sh.nodeKind} ${sh.IRI} ; + ${dash.editor} ${dash.AutoCompleteEditor} ; + ${hydra.search} [ + a ${hydra.IriTemplate} ; + ${hydra.template} "${lindasQueryTemplate(mediaTypesQuery, 'q')}" ; ; + ${hydra.mapping} [ + ${hydra.property} ${hydra.freetextQuery} ; + ${hydra.variable} "q" ; + ${sh.minLength} 3 ; + ] ; + ] ; + ${sh.order} 50 ; + ] , [ + ${sh.name} "Format" ; + ${sh.path} ${dcterms.format} ; + ${sh.maxCount} 1 ; + ${sh.nodeKind} ${sh.IRI} ; + ${dash.editor} ${dash.AutoCompleteEditor} ; + ${hydra.search} [ + a ${hydra.IriTemplate} ; + ${hydra.template} "${lindasQueryTemplate(fileTypesQuery, 'q')}" ; ; + ${hydra.mapping} [ + ${hydra.property} ${hydra.freetextQuery} ; + ${hydra.variable} "q" ; + ${sh.minLength} 3 ; + ] ; + ] ; + ${sh.order} 60 ; + ] , [ + ${sh.path} ${dcat.byteSize} ; + ${sh.name} "Size [bytes]" ; + ${sh.maxCount} 1 ; + ${sh.datatype} ${xsd.decimal} ; + ${sh.order} 70 ; + ] ; . ${temporalFromTo} a ${sh.NodeShape} ; diff --git a/apis/core/package.json b/apis/core/package.json index 1410cf414..84503e145 100644 --- a/apis/core/package.json +++ b/apis/core/package.json @@ -26,7 +26,7 @@ "@sentry/node": "^6.2.0", "@sentry/tracing": "^6.2.0", "@tpluscode/rdf-ns-builders": "^1.0.0", - "@tpluscode/rdf-string": "^0.2.28", + "@tpluscode/rdf-string": "^0.2.29", "@tpluscode/rdfine": "^0.5.43", "@tpluscode/sparql-builder": "^0.3.24", "@uppy/companion": "^4.1.1", diff --git a/cli/.test.env b/cli/.test.env index 418604f7f..fd6563a39 100644 --- a/cli/.test.env +++ b/cli/.test.env @@ -5,13 +5,13 @@ AUTH_RUNNER_CLIENT_ID=cube-creator-runner AUTH_RUNNER_ISSUER=https://keycloak.zazukoians.org/realms/zazuko-dev AUTH_RUNNER_CLIENT_SECRET=private -GRAPH_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/query -GRAPH_STORE_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/data +GRAPH_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/query +GRAPH_STORE_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/data GRAPH_STORE_USER=admin GRAPH_STORE_PASSWORD=password -PUBLISH_GRAPH_STORE_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/data -PUBLISH_GRAPH_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator +PUBLISH_GRAPH_STORE_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/data +PUBLISH_GRAPH_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator PUBLISH_GRAPH_STORE_USER=admin PUBLISH_GRAPH_STORE_PASSWORD=password diff --git a/cli/README.md b/cli/README.md index 637bc3bd1..4a0e567cb 100644 --- a/cli/README.md +++ b/cli/README.md @@ -96,12 +96,12 @@ Here's an example of converting local files using a locally-built image: 1. (optionally) Ensure a fresh container is built ``` - docker-composer build cli + docker compose build cli ``` -1. Run with docker-compose +1. Run with `docker compose` ``` - docker-compose run --rm cli transform \ + docker compose run --rm cli transform \ --to filesystem \ --job --debug diff --git a/e2e-tests/cube-projects/create-invalid.hydra b/e2e-tests/cube-projects/create-invalid.hydra index aaaf6035c..14fe2eb91 100644 --- a/e2e-tests/cube-projects/create-invalid.hydra +++ b/e2e-tests/cube-projects/create-invalid.hydra @@ -122,7 +122,7 @@ With Class api:EntryPoint { <> cc:projectSourceKind ; rdfs:label "Test Project" ; CubeProject:sourceCube ; - CubeProject:sourceEndpoint ; + CubeProject:sourceEndpoint ; schema:maintainer . ``` } => { @@ -142,7 +142,7 @@ With Class api:EntryPoint { <> cc:projectSourceKind ; rdfs:label "Test Project" ; CubeProject:sourceCube ; - CubeProject:sourceEndpoint ; + CubeProject:sourceEndpoint ; schema:maintainer . ``` } => { @@ -162,7 +162,7 @@ With Class api:EntryPoint { <> cc:projectSourceKind ; rdfs:label "Test Project" ; CubeProject:sourceCube ; - CubeProject:sourceEndpoint ; + CubeProject:sourceEndpoint ; schema:maintainer . ``` } => { diff --git a/fuseki/sample-px.trig b/fuseki/sample-px.trig index 39564978d..7211799f3 100644 --- a/fuseki/sample-px.trig +++ b/fuseki/sample-px.trig @@ -31,7 +31,7 @@ graph { cc:projectSourceKind ; CubeProject:sourceCube ; CubeProject:sourceGraph ; - CubeProject:sourceEndpoint ; + CubeProject:sourceEndpoint ; cc:projectDetails ; } @@ -135,6 +135,6 @@ graph { cc:dataset ; CubeProject:sourceCube ; CubeProject:sourceGraph ; - CubeProject:sourceEndpoint ; + CubeProject:sourceEndpoint ; . } diff --git a/package.json b/package.json index 07d3073be..b31ad6f6a 100644 --- a/package.json +++ b/package.json @@ -7,14 +7,14 @@ "lint": "eslint . --ext .ts,.vue,.tsx --quiet --ignore-path .gitignore", "c8": "c8 --reporter lcovonly", "test": "yarn c8 -o coverage/apis mocha --recursive apis/**/*.test.ts packages/**/*.test.ts --grep @SPARQL --invert", - "test:queries": "yarn c8 -o coverage/queries mocha --recursive apis/**/*.test.ts ui/**/*.test.ts --grep @SPARQL", + "test:queries": "yarn c8 -o coverage/queries mocha -t 10000 --recursive apis/**/*.test.ts ui/**/*.test.ts --grep @SPARQL", "test:cli": "run-s test:cli:*", "test:cli:rest": "yarn c8 -o coverage/publish mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/* --invert", "test:cli:import": "yarn c8 -o coverage/publish mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/import", "test:cli:publish": "yarn c8 -o coverage/publish mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/publish", "test:cli:transform": "yarn c8 -o coverage/transform mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/transform", "test:cli:timeoutJobs": "yarn c8 -o coverage/transform mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/timeoutJobs", - "test:e2e": "docker-compose run --rm e2e-tests --", + "test:e2e": "docker compose run --rm e2e-tests --", "seed-data": "dotenv -e .local.env -- bash -c \"ts-node packages/testing/index.ts -i ubd dimensions px-cube hierarchies\"" }, "workspaces": [ diff --git a/packages/testing/lib/index.ts b/packages/testing/lib/index.ts index eaf618ab8..d0b1ee14b 100644 --- a/packages/testing/lib/index.ts +++ b/packages/testing/lib/index.ts @@ -2,9 +2,9 @@ import StreamClient from 'sparql-http-client/StreamClient' import ParsingClient from 'sparql-http-client/ParsingClient' const endpoints = (db: 'cube-creator' | 'shared-dimensions') => ({ - updateUrl: `http://db.cube-creator.lndo.site/${db}/update`, - endpointUrl: `http://db.cube-creator.lndo.site/${db}/query`, - storeUrl: `http://db.cube-creator.lndo.site/${db}/data`, + updateUrl: `https://db.cube-creator.lndo.site/${db}/update`, + endpointUrl: `https://db.cube-creator.lndo.site/${db}/query`, + storeUrl: `https://db.cube-creator.lndo.site/${db}/data`, }) export const ccClients = { diff --git a/ui/src/forms/editors/TextFieldWithLangEditor.vue b/ui/src/forms/editors/TextFieldWithLangEditor.vue index 402646b5a..1367cac5e 100644 --- a/ui/src/forms/editors/TextFieldWithLangEditor.vue +++ b/ui/src/forms/editors/TextFieldWithLangEditor.vue @@ -63,7 +63,7 @@ export default defineComponent({ diff --git a/ui/src/views/CubeMetadataEdit.vue b/ui/src/views/CubeMetadataEdit.vue index 4c05bd70b..72666c034 100644 --- a/ui/src/views/CubeMetadataEdit.vue +++ b/ui/src/views/CubeMetadataEdit.vue @@ -1,5 +1,5 @@