Merge pull request #1565 from zazuko/dim-search

Shared Dimensions: searching and paging

.changeset/late-cows-impress.md

@@ -0,0 +1,6 @@
+---
+"@cube-creator/ui": minor
+"@cube-creator/shared-dimensions-api": minor
+---
+
+Searching and paging Shared Dimensions

apis/shared-dimensions/bootstrap/entrypoint.ts

@@ -5,5 +5,5 @@ import type { BootstrappedResourceFactory } from './index'
 
 export const entrypoint = (ptr: BootstrappedResourceFactory, ns: NamespaceBuilder) =>
   ptr('').addOut(rdf.type, [hydra.Resource, md.Entrypoint])
-    .addOut(md.sharedDimensions, ns('_term-sets?pageSize=1000'))
+    .addOut(md.sharedDimensions, ns('_term-sets?pageSize=20'))
     .addOut(md.hierarchies, ns('_hierarchies'))

apis/shared-dimensions/bootstrap/shapes.ts

@@ -11,6 +11,10 @@ const SharedDimensionUpdate = clownface({ dataset: $rdf.dataset() })
   .namedNode(shape('shape/shared-dimension-update'))
   .addOut(rdf.type, sh.NodeShape)
 
+const SharedDimensionSearch = clownface({ dataset: $rdf.dataset() })
+  .namedNode(shape('shape/shared-dimension-search'))
+  .addOut(rdf.type, sh.NodeShape)
+
 const SharedDimensionTermCreate = clownface({ dataset: $rdf.dataset() })
   .namedNode(shape('shape/shared-dimension-term-create'))
   .addOut(rdf.type, sh.NodeShape)
@@ -30,6 +34,7 @@ const HierarchyCreate = clownface({ dataset: $rdf.dataset() })
 export default [
   SharedDimensionCreate,
   SharedDimensionUpdate,
+  SharedDimensionSearch,
   SharedDimensionTermCreate,
   SharedDimensionTermUpdate,
   Hierarchy,

apis/shared-dimensions/hydra/index.ttl

@@ -1,3 +1,4 @@
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
 BASE <urn:hydra-box:api>
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
 @prefix schema: <http://schema.org/> .
@@ -41,8 +42,10 @@ md:SharedDimensions
   rdfs:subClassOf hydra:Collection ;
   hydra:supportedOperation
     [
-      a hydra:Operation ;
+      a hydra:Operation, schema:DownloadAction ;
       hydra:method "GET" ;
+      hydra:title "Search" ;
+      hydra:expects <dimension/_shape/shared-dimension-search> ;
       code:implementedBy
         [
           a code:EcmaScript ;
@@ -51,7 +54,7 @@ md:SharedDimensions
       hydra-box:variables
         [
           a hydra:IriTemplate ;
-          hydra:template "/_term-sets{?q,pageSize,page}" ;
+          hydra:template "/_term-sets{?q,pageSize,page,includeDeprecated}" ;
           hydra:mapping
             [
               a hydra:IriTemplateMapping ;
@@ -67,6 +70,11 @@ md:SharedDimensions
               a hydra:IriTemplateMapping ;
               hydra:property hydra:pageIndex ;
               hydra:variable "page" ;
+            ],
+            [
+              a hydra:IriTemplateMapping ;
+              hydra:property owl:deprecated ;
+              hydra:variable "includeDeprecated" ;
             ] ;
         ] ;
     ], [
@@ -85,6 +93,7 @@ md:SharedDimensions
 
 <dimension/_shape/shared-dimension-create> a sh:NodeShape, sh:Shape .
 <dimension/_shape/shared-dimension-update> a sh:NodeShape, sh:Shape .
+<dimension/_shape/shared-dimension-search> a sh:NodeShape, sh:Shape .
 
 md:Hierarchies
   a hydra:Class ;

apis/shared-dimensions/lib/domain/hierarchies.ts

@@ -1,4 +1,4 @@
-import type { NamedNode } from '@rdfjs/types'
+import type { NamedNode, Quad } from '@rdfjs/types'
 import { CONSTRUCT, SELECT } from '@tpluscode/sparql-builder'
 import { md, meta } from '@cube-creator/core/namespace'
 import clownface, { GraphPointer } from 'clownface'
@@ -7,9 +7,11 @@ import httpError from 'http-errors'
 import $rdf from 'rdf-ext'
 import slugify from 'slugify'
 import { DomainError } from '@cube-creator/api-errors'
+import { ParsingClient } from 'sparql-http-client/ParsingClient'
 import { SharedDimensionsStore } from '../store'
 import env from '../env'
 import { textSearch } from '../query'
+import { CollectionData } from '../handlers/collection'
 import { newId, replace } from './resource'
 
 interface GetHierarchies {
@@ -18,7 +20,7 @@ interface GetHierarchies {
   offset: number
 }
 
-export function getHierarchies({ freetextQuery, limit, offset }: GetHierarchies) {
+export async function getHierarchies({ freetextQuery, limit, offset }: GetHierarchies, client: ParsingClient): Promise<CollectionData<Iterable<Quad>>> {
   const hierarchy = $rdf.variable('hierarchy')
   const name = $rdf.variable('name')
 
@@ -35,18 +37,21 @@ export function getHierarchies({ freetextQuery, limit, offset }: GetHierarchies)
       .ORDER().BY(name)
   }
 
-  return CONSTRUCT`
+  return {
+    members: await CONSTRUCT`
     ?proxyUrl ?p ?o .
   `
-    .WHERE`
+      .WHERE`
       {
         ${select}
       }
 
       ${hierarchy} ?p ?o .
 
       BIND(IRI(CONCAT("${env.MANAGED_DIMENSIONS_API_BASE}", "dimension/_hierarchy/proxy?id=", ENCODE_FOR_URI(STR(${hierarchy})))) AS ?proxyUrl)
-    `
+    `.execute(client.query),
+    totalItems: 0,
+  }
 }
 
 interface CreateHierarchy {

apis/shared-dimensions/lib/domain/shared-dimensions.ts

@@ -1,8 +1,8 @@
 import path from 'path'
-import type { Quad, Stream, Term } from '@rdfjs/types'
+import type { Quad, Stream, Term, Literal, NamedNode } from '@rdfjs/types'
 import { hydra, rdf, schema, sh } from '@tpluscode/rdf-ns-builders'
 import $rdf from 'rdf-ext'
-import { toRdf } from 'rdf-literal'
+import { toRdf, fromRdf } from 'rdf-literal'
 import { fromFile } from 'rdf-utils-fs'
 import clownface from 'clownface'
 import { isResource } from 'is-graph-pointer'
@@ -11,38 +11,50 @@ import { ParsingClient } from 'sparql-http-client/ParsingClient'
 import { md } from '@cube-creator/core/namespace'
 import env from '../env'
 import shapeToQuery, { rewriteTemplates } from '../shapeToQuery'
+import { CollectionData } from '../handlers/collection'
 import { getDynamicProperties } from './shared-dimension'
 
 interface GetSharedDimensions {
   freetextQuery?: string
   limit?: number
   offset?: number
+  includeDeprecated?: Literal
 }
 
-export async function getSharedDimensions(client: StreamClient, { freetextQuery = '', limit = 10, offset = 0 }: GetSharedDimensions = {}): Promise<Quad[]> {
+export async function getSharedDimensions(client: StreamClient, { freetextQuery = '', limit = 10, offset = 0, includeDeprecated }: GetSharedDimensions = {}): Promise<CollectionData<Iterable<Quad>>> {
   const { constructQuery } = await shapeToQuery()
 
-  const shape = await loadShape('dimensions-query-shape')
+  const memberQueryShape = await loadShape('dimensions-query-shape', md.MembersQueryShape)
+  const totalQueryShape = await loadShape('dimensions-query-shape', md.CountQueryShape)
 
   const { MANAGED_DIMENSIONS_BASE } = env
   const variables = new Map(Object.entries({
     MANAGED_DIMENSIONS_BASE,
     limit,
     offset,
     freetextQuery,
+    includeDeprecated,
     orderBy: schema.name,
   }))
-  await rewriteTemplates(shape, variables)
+  await rewriteTemplates(memberQueryShape, variables)
+  await rewriteTemplates(totalQueryShape, variables)
 
-  const dataset = await $rdf.dataset().import(await constructQuery(shape).execute(client))
+  const dataset = await $rdf.dataset().import(await client.query.construct(constructQuery(memberQueryShape)))
   clownface({ dataset })
     .has(rdf.type, schema.DefinedTermSet)
     .forEach(termSet => {
       termSet.addOut(md.export, $rdf.namedNode(`${MANAGED_DIMENSIONS_BASE}dimension/_export?dimension=${termSet.value}`))
       termSet.addOut(md.terms, $rdf.namedNode(`${MANAGED_DIMENSIONS_BASE}dimension/_terms?dimension=${termSet.value}`))
     })
 
-  return dataset.toArray()
+  const totalItems = clownface({
+    dataset: await $rdf.dataset().import(await client.query.construct(constructQuery(totalQueryShape))),
+  }).has(hydra.totalItems).out(hydra.totalItems).term as Literal
+
+  return {
+    members: dataset,
+    totalItems: fromRdf(totalItems),
+  }
 }
 
 interface GetSharedTerms {
@@ -53,7 +65,7 @@ interface GetSharedTerms {
   validThrough?: Date
 }
 
-export async function getSharedTerms<C extends StreamClient | ParsingClient>({ sharedDimensions, freetextQuery, validThrough, limit = 10, offset = 0 }: GetSharedTerms, client: C): Promise<C extends StreamClient ? Stream : Quad[]> {
+export async function getSharedTerms<C extends StreamClient | ParsingClient>({ sharedDimensions, freetextQuery, validThrough, limit = 10, offset = 0 }: GetSharedTerms, client: C): Promise<CollectionData<C extends StreamClient ? Stream : Quad[]>> {
   const shape = await loadShape('terms-query-shape')
 
   shape.addOut(sh.targetNode, sharedDimensions)
@@ -86,17 +98,19 @@ export async function getSharedTerms<C extends StreamClient | ParsingClient>({ s
   }
 
   const { constructQuery } = await shapeToQuery()
-  return constructQuery(shape).execute(client, {
-    operation: 'postDirect',
-  }) as any
+  return {
+    members: await client.query.construct(constructQuery(shape), {
+      operation: 'postDirect',
+    }) as any,
+  }
 }
 
-async function loadShape(shape: string) {
+async function loadShape(shape: string, shapeType: NamedNode = sh.NodeShape) {
   const dataset = await $rdf.dataset().import(fromFile(path.resolve(__dirname, `../shapes/${shape}.ttl`)))
 
   const ptr = clownface({
     dataset,
-  }).has(rdf.type, sh.NodeShape)
+  }).has(rdf.type, shapeType)
 
   if (!isResource(ptr)) {
     throw new Error('Expected a single blank node or named node')

apis/shared-dimensions/lib/handlers/collection.ts

@@ -1,26 +1,36 @@
-import type { NamedNode, Quad } from '@rdfjs/types'
+import type { NamedNode, Quad, Stream } from '@rdfjs/types'
 import $rdf from 'rdf-ext'
 import clownface, { GraphPointer } from 'clownface'
 import { hydra, rdf } from '@tpluscode/rdf-ns-builders'
 
+export interface CollectionData<M extends Stream | Iterable<Quad> = Stream | Iterable<Quad>> {
+  members: M
+  totalItems?: number
+}
+
 interface CollectionHandler {
   memberType: NamedNode
   collectionType: NamedNode
   view?: NamedNode
-  memberQuads: Quad[]
+  data: CollectionData<Iterable<Quad>>
   collection: NamedNode
 }
 
-export function getCollection({ collection, view, memberQuads, memberType, collectionType }: CollectionHandler): GraphPointer<NamedNode> {
-  const dataset = $rdf.dataset(memberQuads)
+export function getCollection({ collection, view, data: { members: memberQuads, totalItems }, memberType, collectionType }: CollectionHandler): GraphPointer<NamedNode> {
+  const dataset = $rdf.dataset([...memberQuads])
 
   const graph = clownface({ dataset })
   const members = graph.has(rdf.type, memberType)
 
   graph.node(collection)
     .addOut(rdf.type, [hydra.Collection, collectionType])
     .addOut(hydra.member, members)
-    .addOut(hydra.totalItems, members.terms.length)
+
+  if (totalItems) {
+    graph.node(collection).addOut(hydra.totalItems, totalItems)
+  } else {
+    graph.node(collection).addOut(hydra.totalItems, members.terms.length)
+  }
 
   if (view) {
     graph.node(view)

apis/shared-dimensions/lib/handlers/hierarchies.ts

@@ -27,7 +27,7 @@ export const get = asyncMiddleware(async (req, res, next) => {
     offset,
   }
   const collection = await getCollection({
-    memberQuads: await getHierarchies(queryParams).execute(parsingClient.query),
+    data: await getHierarchies(queryParams, parsingClient),
     collectionType: md.Hierarchies,
     memberType: md.Hierarchy,
     collection: req.hydra.resource.term,

apis/shared-dimensions/lib/handlers/hierarchy.ts

@@ -56,7 +56,7 @@ export const getExternal = asyncMiddleware(async (req, res) => {
   })
 
   const hierarchy = clownface({
-    dataset: $rdf.dataset(await query.execute(parsingClient)),
+    dataset: $rdf.dataset(await parsingClient.query.construct(query)),
   }).namedNode(url)
   ensureEndpoint(hierarchy)
 

apis/shared-dimensions/lib/handlers/shared-dimensions.ts

@@ -1,12 +1,12 @@
 import type { Term } from '@rdfjs/types'
-import { hydra, oa, schema } from '@tpluscode/rdf-ns-builders'
+import { hydra, oa, owl, schema, xsd } from '@tpluscode/rdf-ns-builders'
 import { asyncMiddleware } from 'middleware-async'
 import { protectedResource } from '@hydrofoil/labyrinth/resource'
 import { Enrichment } from '@hydrofoil/labyrinth/lib/middleware/preprocessResource'
 import httpError from 'http-errors'
 import clownface, { GraphPointer } from 'clownface'
 import $rdf from 'rdf-ext'
-import { md } from '@cube-creator/core/namespace'
+import * as ns from '@cube-creator/core/namespace'
 import conditional from 'express-conditional-middleware'
 import { isMultipart } from '@cube-creator/express/multipart'
 import { shaclValidate } from '../middleware/shacl'
@@ -29,19 +29,26 @@ export const get = asyncMiddleware(async (req, res, next) => {
   const offset = (page - 1) * pageSize
   const queryParams = {
     freetextQuery: query.has(hydra.freetextQuery).out(hydra.freetextQuery).value,
-    validThrough: query.has(md.onlyValidTerms, query.literal(true)).terms.length ? new Date() : undefined,
+    validThrough: query.has(ns.md.onlyValidTerms, query.literal(true)).terms.length ? new Date() : undefined,
     limit: pageSize,
     offset,
+    includeDeprecated: $rdf.literal(query.has(owl.deprecated).out(owl.deprecated).value || 'false', xsd.boolean),
   }
 
   const collection = getCollection({
     view: $rdf.namedNode(req.absoluteUrl()),
-    memberQuads: await getSharedDimensions(streamClient, queryParams),
-    collectionType: md.SharedDimensions,
+    data: await getSharedDimensions(streamClient, queryParams),
+    collectionType: ns.md.SharedDimensions,
     memberType: schema.DefinedTermSet,
     collection: req.hydra.resource.term,
   })
 
+  collection.addOut(ns.query.templateMappings, templateMappings => {
+    for (const { predicate, object } of query.dataset) {
+      templateMappings.addOut(predicate, object)
+    }
+  })
+
   return res.dataset(collection.dataset)
 })
 
@@ -83,15 +90,15 @@ export const getTerms = asyncMiddleware(async (req, res, next) => {
   const queryParams = {
     sharedDimensions: sharedDimensions.map(rewriteTerm),
     freetextQuery: query.has(hydra.freetextQuery).out(hydra.freetextQuery).value,
-    validThrough: query.has(md.onlyValidTerms, query.literal(true)).terms.length ? new Date() : undefined,
+    validThrough: query.has(ns.md.onlyValidTerms, query.literal(true)).terms.length ? new Date() : undefined,
     limit: pageSize,
     offset,
   }
 
   const collection = getCollection({
-    memberQuads: await getSharedTerms(queryParams, parsingClient),
+    data: await getSharedTerms(queryParams, parsingClient),
     memberType: schema.DefinedTerm,
-    collectionType: md.SharedDimensionTerms,
+    collectionType: ns.md.SharedDimensionTerms,
     collection: termsCollectionId(sharedDimensions, queryParams.freetextQuery),
   })
 
@@ -119,9 +126,9 @@ const postDirect = protectedResource(shaclValidate, asyncMiddleware(async (req,
 export const post = conditional(isMultipart, postImportedDimension, postDirect)
 
 export const injectTermsLink: Enrichment = async (req, pointer) => {
-  pointer.deleteOut(md.terms).addOut(md.terms, termsCollectionId([pointer.term]))
+  pointer.deleteOut(ns.md.terms).addOut(ns.md.terms, termsCollectionId([pointer.term]))
 }
 
 export const injectExportLink: Enrichment = async (req, pointer) => {
-  pointer.deleteOut(md.export).addOut(md.export, pointer.namedNode(`${env.MANAGED_DIMENSIONS_BASE}dimension/_export?dimension=${pointer.value}`))
+  pointer.deleteOut(ns.md.export).addOut(ns.md.export, pointer.namedNode(`${env.MANAGED_DIMENSIONS_BASE}dimension/_export?dimension=${pointer.value}`))
 }

apis/shared-dimensions/lib/namespace.ts

@@ -3,6 +3,7 @@ import env from './env'
 
 type Shapes = 'shape/shared-dimension-create'
 | 'shape/shared-dimension-update'
+| 'shape/shared-dimension-search'
 | 'shape/shared-dimension-term-create'
 | 'shape/shared-dimension-term-update'
 | 'shape/hierarchy-create'