Remove hardcoded paths (#93)

* remove hardcoded path * implement filesystems and sources * rename Hyparam to Hyperparam * details
hyparam · Dec 2, 2024 · 2c17bb1 · 2c17bb1
1 parent 1088c5b
commit 2c17bb1
Show file tree

Hide file tree

Showing 38 changed files with 678 additions and 441 deletions.
diff --git a/.github/workflows/deploy_pages.yml b/.github/workflows/deploy_pages.yml
@@ -29,12 +29,12 @@ jobs:
       - name: Move the build outputs to a folder
         run: |
           mkdir -p build_outputs_folder/apps
-          echo "<h1 id="hyparam">Hyparam</h1>" > build_outputs_folder/index.html
+          echo "<h1 id="hyparam">Hyperparam</h1>" > build_outputs_folder/index.html
           echo "<ul>" >> build_outputs_folder/index.html
           echo "<li><a href="./apps/hyparquet-demo">hyparquet demo</a></li>" >> build_outputs_folder/index.html
           echo "<li><a href="./apps/hightable-demo">hightable demo</a></li>" >> build_outputs_folder/index.html
           echo "</ul>" >> build_outputs_folder/index.html
-          echo "<h1 id="hyparam">Hyparam</h1>" > build_outputs_folder/apps/index.html
+          echo "<h1 id="hyparam">Hyperparam</h1>" > build_outputs_folder/apps/index.html
           echo "<ul>" >> build_outputs_folder/apps/index.html
           echo "<li><a href="./hyparquet-demo">hyparquet demo</a></li>" >> build_outputs_folder/apps/index.html
           echo "<li><a href="./hightable-demo">hightable demo</a></li>" >> build_outputs_folder/apps/index.html

diff --git a/packages/components/demo/App.tsx b/packages/components/demo/App.tsx
@@ -0,0 +1,55 @@
+import React from 'react'
+import { Page } from '../src/index.js'
+
+import { HttpFileSystem } from '../src/index.ts'
+import { Source } from '../src/lib/filesystem.js'
+export interface Navigation {
+  col?: number
+  row?: number
+}
+
+function getNumberParam(search: URLSearchParams, key: string): number | undefined {
+  const value = search.get(key)
+  if (value === null) return
+  const number = Number(value)
+  if (isNaN(number)) return
+  return number
+}
+
+const fileSystems = [
+  new HttpFileSystem(),
+]
+
+export default function App() {
+  const search = new URLSearchParams(location.search)
+  const url = search.get('url')
+  const defaultUrl = '/?url=https://huggingface.co/datasets/severo/test-parquet/resolve/main/parquet/csv-train-00000-of-00001.parquet'
+  if (!url) {
+    location.href = defaultUrl
+    return <div>Redirecting...</div>
+  }
+  // row, col from url
+  const row = getNumberParam(search, 'row')
+  const col = getNumberParam(search, 'col')
+
+  let source: Source | undefined = undefined
+  for (const fileSystem of fileSystems) {
+    const fsSource = fileSystem.getSource(url)
+    if (fsSource){
+      source = fsSource
+      break
+    }
+  }
+
+  if (!source) {
+    return <div>Could not load a data source. You have to pass a valid source in the url, eg: <a href={defaultUrl}>{defaultUrl}</a>.</div>
+  }
+  return <Page source={source} navigation={{ row, col }} config={{
+    slidePanel: { minWidth: 250, maxWidth: 750 },
+    routes: {
+      getSourceRouteUrl: ({ source }) => `/?url=${source}`,
+      getCellRouteUrl: ({ source, col, row }) => `/?url=${source}&col=${col}&row=${row}`,
+    },
+  }} />
+}
+
diff --git a/packages/components/demo/demo.ts b/packages/components/demo/demo.ts
@@ -0,0 +1,9 @@
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import App from './App.tsx'
+
+const app = document.getElementById('app')
+if (!app) throw new Error('missing app element')
+
+const root = ReactDOM.createRoot(app)
+root.render(React.createElement(App))
diff --git a/packages/components/demo/demo.tsx b/packages/components/demo/demo.tsx
diff --git a/packages/components/demo/favicon.png b/packages/components/demo/favicon.png
diff --git a/packages/components/demo/file.svg b/packages/components/demo/file.svg
diff --git a/packages/components/demo/folder.svg b/packages/components/demo/folder.svg
diff --git a/packages/components/demo/index.html b/packages/components/demo/index.html
@@ -6,12 +6,13 @@
     <link rel="stylesheet" href="./styles.css">
     <link rel="stylesheet" href="./HighTable.css">
     <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Mulish:wght@400;600&display=swap"/>
-    <meta name="description" content="demo for the Hyparam components" />
+    <link href="favicon.png" rel="icon" />
+    <meta name="description" content="demo for the Hyperparam components" />
     <meta name="theme-color" content="#6b00ff">
     <meta name="viewport" content="width=device-width, initial-scale=1" />
   </head>
   <body>
     <div id="app"></div>
-    <script type="module" src="./demo.tsx"></script>
+    <script type="module" src="./demo.ts"></script>
   </body>
 </html>
diff --git a/packages/components/demo/logo.svg b/packages/components/demo/logo.svg
diff --git a/packages/components/demo/styles.css b/packages/components/demo/styles.css
@@ -226,14 +226,14 @@ main {
 
 /* file icons */
 .file {
-  background: url(/public/file.svg);
+  background: url(file.svg);
   background-position: left center;
   background-repeat: no-repeat;
   background-size: 12px;
   padding-left: 22px;
 }
 .folder {
-  background-image: url(/public/folder.svg);
+  background-image: url(folder.svg);
 }
 
 /* viewer */

diff --git a/packages/components/src/components/App.tsx b/packages/components/src/components/App.tsx
diff --git a/packages/components/src/components/Breadcrumb.tsx b/packages/components/src/components/Breadcrumb.tsx
@@ -1,37 +1,22 @@
-import { FileKey, UrlKey } from '../lib/key.js'
+import { Source } from '../lib/filesystem.js'
+import { RoutesConfig } from '../lib/routes.js'
 
+export type BreadcrumbConfig = RoutesConfig
 interface BreadcrumbProps {
-  parsedKey: UrlKey | FileKey
+  source: Source,
+  config?: BreadcrumbConfig
 }
 
-function UrlBreadcrumb({ urlKey }: { urlKey: UrlKey}) {
-  return <nav className='top-header'>
-    <div className='path'>
-      <a href={`/files?key=${urlKey.raw}`}>{urlKey.raw}</a>
-    </div>
-  </nav>
-}
-
-function FileBreadcrumb({ fileKey }: { fileKey: FileKey}) {
-  const path = fileKey.raw.split('/')
-
+/**
+ * Breadcrumb navigation
+ */
+export default function Breadcrumb({ source, config }: BreadcrumbProps) {
   return <nav className='top-header'>
     <div className='path'>
-      <a href='/files'>/</a>
-      {path.slice(0, -1).map((sub, depth) =>
-        <a href={`/files?key=${path.slice(0, depth + 1).join('/')}/`} key={depth}>{sub}/</a>,
+      {source.getSourceParts().map((part, depth) =>
+        <a href={config?.routes?.getSourceRouteUrl?.({ source: part.source }) ?? ''} key={depth}>{part.name}</a>,
       )}
-      <a href={`/files?key=${fileKey.raw}`}>{fileKey.fileName}</a>
     </div>
   </nav>
-}
 
-/**
- * Breadcrumb navigation
- */
-export default function Breadcrumb({ parsedKey }: BreadcrumbProps) {
-  if (parsedKey.kind === 'url') {
-    return <UrlBreadcrumb urlKey={parsedKey} />
-  }
-  return <FileBreadcrumb fileKey={parsedKey} />
 }
diff --git a/packages/components/src/components/Cell.tsx b/packages/components/src/components/Cell.tsx
@@ -1,15 +1,18 @@
 import { asyncRows } from 'hightable'
 import { asyncBufferFromUrl, parquetMetadataAsync } from 'hyparquet'
 import { useEffect, useState } from 'react'
-import { FileKey, UrlKey } from '../lib/key.js'
+import { FileSource } from '../lib/filesystem.js'
 import { parquetDataFrame } from '../lib/tableProvider.js'
-import Breadcrumb from './Breadcrumb.js'
+import Breadcrumb, { BreadcrumbConfig } from './Breadcrumb.js'
 import Layout from './Layout.js'
 
+export type CellConfig = BreadcrumbConfig
+
 interface CellProps {
-  parsedKey: FileKey | UrlKey;
+  source: FileSource;
   row: number;
   col: number;
+  config?: CellConfig
 }
 
 enum LoadingState {
@@ -21,14 +24,14 @@ enum LoadingState {
 /**
  * Cell viewer displays a single cell from a table.
  */
-export default function CellView({ parsedKey, row, col }: CellProps) {
+export default function CellView({ source, row, col, config }: CellProps) {
   const [loading, setLoading] = useState<LoadingState>(LoadingState.NotLoaded)
   const [text, setText] = useState<string | undefined>()
   const [progress, setProgress] = useState<number>()
   const [error, setError] = useState<Error>()
 
   // File path from url
-  const { resolveUrl, fileName } = parsedKey
+  const { resolveUrl, fileName } = source
 
   // Load cell data
   useEffect(() => {
@@ -67,7 +70,7 @@ export default function CellView({ parsedKey, row, col }: CellProps) {
 
   return (
     <Layout progress={progress} error={error} title={fileName}>
-      <Breadcrumb parsedKey={parsedKey} />
+      <Breadcrumb source={source} config={config} />
 
       {/* <Highlight text={text || ''} /> */}
       <pre className="viewer text">{text}</pre>

diff --git a/packages/components/src/components/File.tsx b/packages/components/src/components/File.tsx
@@ -1,25 +1,25 @@
 import { useState } from 'react'
-import { FileKey, UrlKey } from '../lib/key.js'
-import Breadcrumb from './Breadcrumb.js'
+import { FileSource } from '../lib/filesystem.js'
+import Breadcrumb, { BreadcrumbConfig } from './Breadcrumb.js'
 import Layout from './Layout.js'
 import Viewer, { ViewerConfig } from './viewers/Viewer.js'
 
-export type FileConfig = ViewerConfig
+export type FileConfig = ViewerConfig & BreadcrumbConfig
 
 interface FileProps {
-  parsedKey: UrlKey | FileKey
+  source: FileSource
   config?: FileConfig
 }
 
 /**
  * File viewer page
  */
-export default function File({ parsedKey, config }: FileProps) {
+export default function File({ source, config }: FileProps) {
   const [progress, setProgress] = useState<number>()
   const [error, setError] = useState<Error>()
 
-  return <Layout progress={progress} error={error} title={parsedKey.fileName}>
-    <Breadcrumb parsedKey={parsedKey} />
-    <Viewer parsedKey={parsedKey} setProgress={setProgress} setError={setError} config={config} />
+  return <Layout progress={progress} error={error} title={source.fileName}>
+    <Breadcrumb source={source} config={config} />
+    <Viewer source={source} setProgress={setProgress} setError={setError} config={config} />
   </Layout>
 }
diff --git a/packages/components/src/components/Folder.tsx b/packages/components/src/components/Folder.tsx
@@ -1,59 +1,47 @@
-import { useCallback, useEffect, useState } from 'react'
-import { FileMetadata, getFileDate, getFileDateShort, getFileSize, listFiles } from '../lib/files.js'
-import { FolderKey } from '../lib/key.js'
-import { cn } from '../lib/utils.js'
+import { useEffect, useState } from 'react'
+import { DirSource, FileMetadata } from '../lib/filesystem.js'
+import { cn, formatFileSize, getFileDate, getFileDateShort } from '../lib/utils.js'
+import Breadcrumb, { BreadcrumbConfig } from './Breadcrumb.js'
 import Layout, { Spinner } from './Layout.js'
 
+export type FolderConfig = BreadcrumbConfig
+
 interface FolderProps {
-  folderKey: FolderKey
+  source: DirSource
+  config?: FolderConfig
 }
 
 /**
  * Folder browser page
  */
-export default function Folder({ folderKey }: FolderProps) {
+export default function Folder({ source, config }: FolderProps) {
   // State to hold file listing
   const [files, setFiles] = useState<FileMetadata[]>()
   const [error, setError] = useState<Error>()
 
-  // Folder path from url
-  const { prefix, listFilesUrl } = folderKey
-  const path = prefix.split('/')
-
   // Fetch files on component mount
   useEffect(() => {
-    listFiles(listFilesUrl)
+    source.listFiles()
       .then(setFiles)
       .catch((error: unknown) => {
         setFiles([])
         setError(error instanceof Error ? error : new Error(`Failed to fetch files - ${error}`))
       })
-  }, [listFilesUrl])
-
-  const fileUrl = useCallback((file: FileMetadata) => {
-    return prefix ? `/files?key=${prefix}/${file.key}` : `/files?key=${file.key}`
-  }, [prefix])
+  }, [source])
 
-  return <Layout error={error} title={prefix}>
-    <nav className='top-header'>
-      <div className='path'>
-        <a href='/files'>/</a>
-        {prefix.length > 0 && prefix.split('/').map((sub, depth) =>
-          <a href={`/files?key=${path.slice(0, depth + 1).join('/')}/`} key={depth}>{sub}/</a>,
-        )}
-      </div>
-    </nav>
+  return <Layout error={error} title={source.prefix}>
+    <Breadcrumb source={source} config={config} />
 
     {files && files.length > 0 && <ul className='file-list'>
       {files.map((file, index) =>
         <li key={index}>
-          <a href={fileUrl(file)}>
-            <span className={cn('file-name', 'file', file.key.endsWith('/') && 'folder')}>
-              {file.key}
+          <a href={config?.routes?.getSourceRouteUrl?.({ source: file.source }) ?? location.href}>
+            <span className={cn('file-name', 'file', file.kind === 'directory' && 'folder')}>
+              {file.name}
             </span>
-            {!file.key.endsWith('/') && <>
-              {file.fileSize !== undefined && <span className='file-size' title={file.fileSize.toLocaleString() + ' bytes'}>
-                {getFileSize(file)}
+            {file.kind === 'file' && <>
+              {file.size !== undefined && <span className='file-size' title={file.size.toLocaleString() + ' bytes'}>
+                {formatFileSize(file.size)}
               </span>}
               <span className='file-date' title={getFileDate(file)}>
                 {getFileDateShort(file)}