Merge pull request #179 from hypermedia-app/uri-resolution

refactor: URI resolution

.changeset/mighty-frogs-thank.md

@@ -0,0 +1,8 @@
+---
+"@hydrofoil/talos-core": minor
+"@hydrofoil/talos": minor
+---
+
+SPARQL Queries are adjusted to use the base URI calculated from the resource path. For example, in query `/tables/generate.ru`,
+the effective base URI would be `/tables/generate/`. This is to align this behavior with how static sources are parsed.
+In such case, rename the file to `index.ru` to remove the file name from resolves URIs.

.changeset/popular-apricots-flow.md

@@ -0,0 +1,6 @@
+---
+"@hydrofoil/talos-core": minor
+"@hydrofoil/talos": minor
+---
+
+Ensures trailing slash in bare-domain resources

.changeset/spicy-phones-tease.md

@@ -0,0 +1,19 @@
+---
+"@hydrofoil/talos-core": minor
+"@hydrofoil/talos": minor
+---
+
+Base URI behavior changed. Now relative URIs will be resolved against the calculated base including a trailing slash.
+The exception is an empty `<>` reference which will be resolved against the base without a trailing slash.
+Use `<./>` to create a resource with a trailing slash.
+
+| File path         | URI reference | Resulting URI      |
+|-------------------|---------------|--------------------|
+| `/api/people.ttl` | `<>`          | `/api/people`      |
+| `/api/people.ttl` | `<.>`         | `/api/people`      |
+| `/api/people.ttl` | `<./>`        | `/api/people/`     |
+| `/api/people.ttl` | `<john>`      | `/api/people/john` |
+| `/api/people.ttl` | `<#john>`     | `/api/people#john` |
+| `/api/people.ttl` | `<../people>` | `/api/people`      |
+| `/api/people.ttl` | `<./people>`  | `/api/people`      |
+| `/api/people.ttl` | `</projects>` | `/projects`        |

.changeset/unlucky-dolphins-care.md

@@ -0,0 +1,6 @@
+---
+"@hydrofoil/talos-core": patch
+"@hydrofoil/talos": patch
+---
+
+Trailing slash in base URI is truncated when resolving relative URI references

packages/cli/test/lib/command/__snapshots__/put.test.ts.snap

@@ -8,7 +8,7 @@ exports[`@hydrofoil/talos @hydrofoil/talos/lib/command/put --resources base http
 `;
 
 exports[`@hydrofoil/talos @hydrofoil/talos/lib/command/put --resources base http://example.com turtle replaces entire graph by default 1`] = `
-"<http://example.com/project> <http://schema.org/parentItem> <http://example.com> .
+"<http://example.com/project> <http://schema.org/parentItem> <http://example.com/> .
 <http://example.com/project> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://schema.org/Thing> .
 "
 `;

packages/cli/test/lib/command/put.test.ts

@@ -13,15 +13,15 @@ import { put } from '../../../lib/command/put.js'
 const testResources = url.fileURLToPath(new URL('../../../../../test-resources', import.meta.url))
 
 const apis = [
-  'http://example.com',
-  'http://example.com/base',
+  ['http://example.com', 'http://example.com/'],
+  ['http://example.com/base', 'http://example.com/base'],
 ]
 const dir = path.resolve(testResources, './resources')
 
 describe('@hydrofoil/talos', function () {
   this.timeout(20000)
 
-  for (const base of apis) {
+  for (const [base, rootResource] of apis) {
     use(jestSnapshotPlugin())
 
     const ns = $rdf.namespace(base + '/')
@@ -176,19 +176,19 @@ template
             await expect(indexCorrectlyInserted).to.eventually.be.true
           })
 
-          it('does not generated trailing slash for root handles index.ttl', async () => {
+          it('correctly resolves root resource from index.ttl', async () => {
             const indexCorrectlyInserted = ASK`
-            ${$rdf.namedNode(base)} a ${schema.Thing}
+            ${$rdf.namedNode(rootResource)} a ${schema.Thing}
           `
-              .FROM($rdf.namedNode(base))
+              .FROM($rdf.namedNode(rootResource))
               .execute(client)
 
             await expect(indexCorrectlyInserted).to.eventually.be.true
           })
 
-          it('removes trailing slash from relative paths resulting in root URI', async () => {
+          it('keep trailing slash from relative paths resulting in root URI', async () => {
             const indexCorrectlyInserted = ASK`
-            ${ns('project')} ${schema.parentItem} <${base}>
+            ${ns('project')} ${schema.parentItem} <${rootResource}>
           `
               .FROM(ns('project'))
               .execute(client)
@@ -309,7 +309,7 @@ template
 
         it('merges statements from multiple graph documents', async () => {
           const ask = ASK`
-          <${base}>
+          <${rootResource}>
             ${schema.name} "Bar environment" ;
             ${schema.hasPart} [
               ${schema.minValue} 10 ;

packages/core/index.ts

@@ -7,8 +7,8 @@ import $rdf from './env.js'
 import log from './lib/log.js'
 import { getPatchedStream } from './lib/fileStream.js'
 import { optionsFromPrefixes } from './lib/prefixHandler.js'
-import { baseIRI } from './lib/baseIRI.js'
 import { applyUpdates } from './lib/applyUpdates.js'
+import { resourcePathFromFilePath } from './lib/iri.js'
 
 const { talos: ns } = $rdf.ns
 export { ns }
@@ -23,9 +23,10 @@ interface Options {
 }
 
 export async function fromDirectories(directories: string[], baseIri: string, { includeMetaGraph }: Options = { includeMetaGraph: true }): Promise<Dataset> {
+  const baseIriNoSlash = baseIri.replace(/\/$/, '')
   const validDirs = directories.filter(isValidDir)
-  const dataset = await validDirs.reduce(toGraphs(baseIri), Promise.resolve($rdf.dataset()))
-  const updatedDataset = await applyUpdates(baseIri, validDirs, dataset)
+  const dataset = await validDirs.reduce(toGraphs(new URL(baseIri)), Promise.resolve($rdf.dataset()))
+  const updatedDataset = await applyUpdates(baseIriNoSlash, validDirs, dataset)
 
   setDefaultAction(updatedDataset)
 
@@ -50,7 +51,8 @@ function setDefaultAction(dataset: Dataset) {
     .addOut($rdf.ns.talos.action, $rdf.ns.talos.overwrite)
 }
 
-function toGraphs(api: string) {
+function toGraphs(baseIri: URL) {
+  const baseIriNoSlash = baseIri.toString().replace(/\/$/, '')
   return async function (previousPromise: Promise<Dataset>, dir: string): Promise<Dataset> {
     let previous = await previousPromise
     const dataset = $rdf.dataset()
@@ -63,9 +65,10 @@ function toGraphs(api: string) {
       }
 
       const relative = path.relative(dir, file)
-      const url = baseIRI(relative, api)
+      const resourcePath = resourcePathFromFilePath(relative)
+      const resourceUrl = $rdf.namedNode(resourcePath === '' ? baseIri.toString() : `${baseIriNoSlash}/${resourcePath}`)
 
-      const parserStream = getPatchedStream(file, dir, api, url)
+      const parserStream = getPatchedStream(file, dir, baseIriNoSlash, resourcePath)
       if (!parserStream) {
         continue
       }
@@ -78,7 +81,7 @@ function toGraphs(api: string) {
       const resourceOptions = $rdf.clownface({ dataset: previous, graph: $rdf.ns.talos.resources })
       try {
         for await (const { subject, predicate, object, ...quad } of parserStream) {
-          const graph: NamedNode = quad.graph.equals($rdf.defaultGraph()) ? $rdf.namedNode(url) : quad.graph
+          const graph: NamedNode = quad.graph.equals($rdf.defaultGraph()) ? resourceUrl : quad.graph
 
           if (!resources.has(graph)) {
             log.debug(`Found graph ${graph.value}`)

packages/core/lib/applyUpdates.ts

@@ -7,9 +7,11 @@ import type { DatasetCore } from '@rdfjs/types'
 import { translate } from 'sparqlalgebrajs'
 import type { Operation } from 'sparqlalgebrajs/lib/algebra.js'
 import { types } from 'sparqlalgebrajs/lib/algebra.js'
+import toString from 'stream-to-string'
 import $rdf from '../env.js'
 import log from './log.js'
-import { baseIRI as getBaseIRI } from './baseIRI.js'
+import { resourcePathFromFilePath } from './iri.js'
+import { angleBracketTransform } from './fileStream.js'
 
 export async function applyUpdates(api: string, validDirs: string[], dataset: DatasetCore) {
   const engine = new QueryEngine()
@@ -22,11 +24,16 @@ export async function applyUpdates(api: string, validDirs: string[], dataset: Da
       }
       const destination = new Store()
       const relative = path.relative(dir, file)
-      const baseIRI = getBaseIRI(relative, api)
+      const resourcePath = resourcePathFromFilePath(relative)
+      const baseIRI = api + '/' + resourcePath + '/'
       log.trace(`Applying updates from ${relative}`)
-      const query = fs.readFileSync(file, 'utf-8')
+      const query = await toString(fs.createReadStream(file, 'utf-8').pipe(angleBracketTransform(api, resourcePath)))
 
+      if (!hasBaseIRI(query)) {
+        log.info(`No BASE clause in ${relative}. Effective base IRI: ${baseIRI}`)
+      }
       const algebra = translate(query, { quads: true, baseIRI })
+
       for (const command of getUpdates(algebra)) {
         await engine.queryVoid(command, {
           sources: [destination, store],
@@ -44,6 +51,10 @@ export async function applyUpdates(api: string, validDirs: string[], dataset: Da
   return result
 }
 
+function hasBaseIRI(query: string) {
+  return query.match(/^(?!\s*#)\s*BASE\s+<[^>]*>/m)
+}
+
 function getUpdates(query: Operation) {
   switch (query.type) {
     case types.COMPOSITE_UPDATE:

packages/core/lib/fileStream.ts

@@ -6,47 +6,33 @@ import replaceStream from 'replacestream'
 import isAbsoluteUrl from 'is-absolute-url'
 import rdf from '../env.js'
 import log from './log.js'
+import { resolveResourceUri } from './streamPatchTransform.js'
+
+function replacer(baseUri: string, resourcePath: string, s: string, e = s) {
+  const resolve = resolveResourceUri(baseUri, resourcePath)
 
-function replacer(basePath: string, resourceUrl: string, s: string, e = s) {
   return (_: unknown, match: string) => {
-    let absolute: string
     if (isAbsoluteUrl(match)) {
       return `${s}${match}${e}`
     }
 
-    if (match.startsWith('/') && basePath !== '/') {
-      absolute = basePath + match
-    } else {
-      absolute = new URL(match, resourceUrl).toString()
-      if (!match.endsWith('/')) {
-        absolute = absolute.replace(/\/?$/, '')
-      }
-    }
-
-    const absoluteUrl = new URL(absolute, resourceUrl)
-    if (absoluteUrl.pathname === '/' && basePath !== '/') {
-      absoluteUrl.pathname = basePath
-      absolute = absoluteUrl.toString()
-    }
-
-    return `${s}${absolute}${e}`
+    return `${s}${resolve(match)}${e}`
   }
 }
 
-const angleBracketTransform = (basePath: string, resourceUrl: string) => replaceStream(/<([^>]+)>(?=([^"\\]*(\\.|"([^"\\]*\\.)*[^"\\]*"))*[^"]*$)/g, replacer(basePath, resourceUrl, '<', '>'))
-const jsonTransform = (basePath: string, resourceUrl: string) => replaceStream(/"([./][^"]+)"/g, replacer(basePath, resourceUrl, '"'))
+export const angleBracketTransform = (baseUri: string, resourcePath: string) => replaceStream(/<([^>]*)>(?=([^"\\]*(\\.|"([^"\\]*\\.)*[^"\\]*"))*[^"]*$)/g, replacer(baseUri, resourcePath, '<', '>'))
+export const jsonTransform = (baseUri: string, resourcePath: string) => replaceStream(/"@id": "([^"]*)"/g, replacer(baseUri, resourcePath, '"@id": "', '"'))
 
-const filePatchTransforms = new Map([
+export const filePatchTransforms = new Map([
   ['text/turtle', angleBracketTransform],
   ['application/n-triples', angleBracketTransform],
   ['application/n-quads', angleBracketTransform],
   ['application/trig', angleBracketTransform],
   ['application/ld+json', jsonTransform],
 ])
 
-export function getPatchedStream(file: string, cwd: string, api: string, resourceUrl: string): Readable | null {
+export function getPatchedStream(file: string, cwd: string, baseIRI: string, resourcePath: string): Readable | null {
   const relative = path.relative(cwd, file)
-  const basePath = new URL(api).pathname
   const mediaType = mime.lookup(file)
   if (!mediaType) {
     log.warn(`Could not determine media type for file ${relative}`)
@@ -55,11 +41,11 @@ export function getPatchedStream(file: string, cwd: string, api: string, resourc
 
   let fileStream = fs.createReadStream(file)
   if (filePatchTransforms.has(mediaType)) {
-    fileStream = fileStream.pipe(filePatchTransforms.get(mediaType)!(basePath, resourceUrl))
+    fileStream = fileStream.pipe(filePatchTransforms.get(mediaType)!(baseIRI, resourcePath))
   }
 
   const parserStream = rdf.formats.parsers.import(mediaType, fileStream, {
-    baseIRI: resourceUrl,
+    baseIRI,
     blankNodePrefix: '',
   })
 

packages/core/lib/iri.ts

@@ -0,0 +1,8 @@
+export function resourcePathFromFilePath(resourcePath: string) {
+  resourcePath = resourcePath
+    .replace(/\.[^.]+$/, '')
+    .replace(/\/?index$/, '')
+  return resourcePath === ''
+    ? ''
+    : encodeURI(resourcePath)
+}

packages/core/lib/streamPatchTransform.ts

@@ -0,0 +1,51 @@
+export function resolveResourceUri(baseUri: string, resourcePath: string) {
+  const baseUriUri = new URL(baseUri)
+  const basePath = baseUriUri.pathname.split('/').filter(Boolean)
+
+  if (resourcePath.endsWith('/')) {
+    throw new Error('Resource URL must not end with a slash')
+  }
+  if (resourcePath.startsWith('/')) {
+    throw new Error('Resource URL must not start with a slash')
+  }
+
+  return (path: string) => {
+    if (path === '/') {
+      return baseUriUri.toString()
+    }
+    if ((path === '' || path === '.') && resourcePath === '') {
+      return baseUriUri.toString()
+    }
+
+    if (path.startsWith('/')) {
+      return new URL('/' + mergePaths(basePath, path.split('/').slice(1)).join('/'), baseUri).toString()
+    }
+
+    const combinedPath = [...basePath, ...resourcePath.split('/').filter(Boolean)]
+    if (path.startsWith('#') || path === '') {
+      const url = new URL('/' + combinedPath.join('/'), baseUri)
+      if (path) {
+        url.hash = path
+      }
+      return url.toString()
+    }
+
+    return new URL('/' + mergePaths(combinedPath, path.split('/')).join('/'), baseUri).toString()
+  }
+}
+
+function mergePaths(basePath: string[], resourcePath: string[]) {
+  const result = basePath.slice()
+  for (const segment of resourcePath) {
+    if (segment === '.') {
+      continue
+    }
+    if (segment === '..') {
+      result.pop()
+      continue
+    }
+    result.push(segment)
+  }
+
+  return result
+}

packages/core/package.json

@@ -25,7 +25,8 @@
     "mime-types": "^2.1.35",
     "n3": "^1.21.0",
     "replacestream": "^4.0.3",
-    "sparqlalgebrajs": "^4.3.8"
+    "sparqlalgebrajs": "^4.3.8",
+    "stream-to-string": "^1.2.1"
   },
   "devDependencies": {
     "@rdfjs-elements/formats-pretty": "^0.6.7",