Merge pull request #130 from hyparam/npm-workspaces-again

move to npm packages again

.github/workflows/_common_jobs.yml

@@ -2,47 +2,35 @@ name: Launch CI jobs for a package or app
 on:
   workflow_call:
     inputs:
-      working-directory:
+      workspace:
         required: true
         type: string
 jobs:
   lint:
-    defaults:
-      run:
-        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: npm i
-      - run: npm run lint
+      - run: npm run lint -w ${{ inputs.workspace }}
 
   typecheck:
-    defaults:
-      run:
-        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: npm i
-      - run: npx tsc
+      - run: npm run typecheck -w ${{ inputs.workspace }}
 
   test:
-    defaults:
-      run:
-        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: npm i
-      - run: npm run coverage
+      - run: npm run coverage -w ${{ inputs.workspace }}
 
   buildcheck:
-    defaults:
-      run:
-        working-directory: ${{ inputs.working-directory }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: npm i
-      - run: npm run build
+      - run: npm run build -w ${{ inputs.workspace }}
 

.github/workflows/check_dependencies.yml

@@ -0,0 +1,16 @@
+name: check dependencies
+on:
+  push:
+    paths:
+      - '**/package.json'
+      - '.github/workflows/check_dependencies.yml'
+jobs:
+  check-dependencies:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: npm i
+      - name: Check if the dependencies are free of any conflict and match "overrides" in the root package.json
+        run: npm ls

.github/workflows/ci_apps_hightable_demo.yml

@@ -5,9 +5,10 @@ on:
       - 'apps/hightable-demo/**'
       - '.github/workflows/_common_jobs.yml'
       - '.github/workflows/ci_apps_hightable_demo.yml'
-      - 'shared.eslint.config.js'
+      - 'eslint.config.js'
+      - 'package.json'
 jobs:
   ci:
     uses: ./.github/workflows/_common_jobs.yml
     with:
-      working-directory: apps/hightable-demo
+      workspace: hightable-demo

.github/workflows/ci_apps_hyparquet_demo.yml

@@ -2,12 +2,24 @@ name: apps/hyparquet-demo
 on:
   push:
     paths:
+      - 'packages/components/**'
       - 'apps/hyparquet-demo/**'
       - '.github/workflows/_common_jobs.yml'
       - '.github/workflows/ci_apps_hyparquet_demo.yml'
-      - 'shared.eslint.config.js'
+      - 'eslint.config.js'
+      - 'package.json'
 jobs:
+  build-components:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: npm i
+      - name: Build components package
+        run: npm run build -w @hyparam/components
   ci:
+    needs: build-components
     uses: ./.github/workflows/_common_jobs.yml
     with:
-      working-directory: apps/hyparquet-demo
+      workspace: hyparquet-demo

.github/workflows/ci_packages_cli.yml

@@ -2,12 +2,24 @@ name: apps/cli
 on:
   push:
     paths:
+      - 'packages/components/**'
       - 'packages/cli/**'
       - '.github/workflows/_common_jobs.yml'
       - '.github/workflows/ci_packages_cli.yml'
-      - 'shared.eslint.config.js'
+      - 'eslint.config.js'
+      - 'package.json'
 jobs:
+  build-components:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: npm i
+      - name: Build components package
+        run: npm run build -w @hyparam/components
   ci:
+    needs: build-components
     uses: ./.github/workflows/_common_jobs.yml
     with:
-      working-directory: packages/cli
+      workspace: hyperparam

.github/workflows/ci_packages_components.yml

@@ -5,9 +5,10 @@ on:
       - 'packages/components/**'
       - '.github/workflows/_common_jobs.yml'
       - '.github/workflows/ci_packages_components.yml'
-      - 'shared.eslint.config.js'
+      - 'eslint.config.js'
+      - 'package.json'
 jobs:
   ci:
     uses: ./.github/workflows/_common_jobs.yml
     with:
-      working-directory: packages/components
+      workspace: '@hyparam/components'

.github/workflows/ci_root.yml

@@ -0,0 +1,22 @@
+name: root
+on:
+  push:
+    paths:
+      - '*'
+      - 'test/**'
+      - '.github/workflows/ci_root.yml'
+      - 'eslint.config.js'
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm i
+      - run: npm run lint
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm i
+      - run: npm run test

.github/workflows/deploy_pages.yml

@@ -4,9 +4,11 @@ on:
   push:
     branches: ["master"]
     paths:
+      - "packages/components/**"
       - "apps/hyparquet-demo/**"
       - "apps/hightable-demo/**"
       - ".github/workflows/deploy_pages.yml"
+      - 'package.json'
   workflow_dispatch:
 
 jobs:
@@ -16,16 +18,12 @@ jobs:
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v4
-      - name: Build hyparquet demo
+      - name: Build components package and demo apps
         run: |
           npm i
-          npm run build
-        working-directory: ./apps/hyparquet-demo
-      - name: Build hightable demo
-        run: |
-          npm i
-          npm run build
-        working-directory: ./apps/hightable-demo
+          npm run build -w @hyparam/components
+          npm run build -w hyparquet-demo
+          npm run build -w hightable-demo
       - name: Move the build outputs to a folder
         run: |
           mkdir -p build_outputs_folder/apps

README.md

@@ -3,15 +3,81 @@
 This is a monorepo for the Hyperparam project.
 
 It contains the following packages, published to npm:
+
 - [`@hyparam/components`](./packages/components): a library of React components and utilities for building Hyperparam UIs.
 - [`hyperparam`](./packages/cli): a cli tool for viewing arbitrarily large datasets in the browser.
 
 It also contains the following applications:
+
 - [`hightable-demo`](./apps/hightable-demo): an example project showing how to use [hightable](https://github.com/hyparam/hightable).
 - [`hyparquet-demo`](./apps/hyparquet-demo): an example project showing how to use [hyparquet](https://github.com/hyparam/hyparquet).
 
+## Use
+
+Install all the workspaces with:
+
+```bash
+npm install
+```
+
+Lint all the workspaces:
+
+```bash
+npm run lint -ws
+```
+
+Test all the workspaces:
+
+```bash
+npm test -ws
+```
+
+Compute the coverage for all the workspaces:
+
+```bash
+npm run coverage -ws
+```
+
+Build all the workspaces (they are built in order, so the dependencies are built first - it's defined manually by the order of the workspaces in the `package.json`):
+
+```bash
+npm run build -ws
+```
+
+Run an app:
+
+- `hightable-demo`:
+
+```bash
+npm run dev -w hightable-demo
+```
+
+- `hyparquet-demo`:
+
+```bash
+npm run dev -w hyparquet-demo
+```
+
+- `hyperparam`:
+
+```bash
+npm run dev -w hyperparam
+```
+
+- `components`:
+
+```bash
+npm run dev -w @hyparam/components
+```
+
 ## Development
 
-The dependencies between the packages and applications of this monorepo are pinned, and the packages are published to npm. It means that the changes in a dependency are not automatically reflected in the dependent package or application.
+The monorepo is managed with [npm workspaces](https://docs.npmjs.com/cli/v10/using-npm/workspaces). Some workspaces are dependencies of others. Currently:
+
+- `@hyparam/components` is a dependency of `hyperparam` and `hyparquet-demo`.
+
+It means that if you make a change to `@hyparam/components`, you need to rebuild it before developing `hyperparam` or `hyparquet-demo`.
+
+Also, if you increment the version of `@hyparam/components`, you need to update the version of `@hyparam/components` in the `package.json` of `hyperparam` and `hyparquet-demo`, as we use exact versions in the `package.json` of the workspaces. Note that we don't have to increment the version after every change, only when we want to publish a new version with a significant batch of changes.
 
-To make the development easier, you can locally replace the npm dependencies with the local packages, 1. creating a local package with `npm pack`, 2. replacing the pinned versions (eg `"@hyparam/components": "0.1.3",`) with a relative path to the local package (`"@hyparam/components": "file:../../packages/components/hyparam-components-0.1.3.tgz",`).
+The root package.json contains a special field, `overrides`, which sets the shared dependencies versions we want to have in all the workspaces. The "check_dependencies" GitHub action checks that the dependencies are the same in all the workspaces (`npm ls` would fail if dependency version mismatch).

apps/hightable-demo/eslint.config.js

@@ -4,8 +4,7 @@ import reactHooks from 'eslint-plugin-react-hooks'
 import reactRefresh from 'eslint-plugin-react-refresh'
 import globals from 'globals'
 import tseslint from 'typescript-eslint'
-import sharedEslintRules from '../../shared.eslint.config.js'
-const { sharedJsRules, sharedTsRules } = sharedEslintRules
+import { sharedJsRules, sharedTsRules } from '../../eslint.config.js'
 
 export default tseslint.config(
   { ignores: ['dist', 'coverage/'] },

apps/hightable-demo/package.json

@@ -10,27 +10,12 @@
     "build": "tsc -b && vite build",
     "lint": "eslint .",
     "preview": "vite preview",
-    "test": "vitest run --dir test"
+    "test": "vitest run --dir test",
+    "typecheck": "tsc"
   },
   "dependencies": {
     "hightable": "0.7.2",
     "react": "18.3.1",
     "react-dom": "18.3.1"
-  },
-  "devDependencies": {
-    "@eslint/js": "9.16.0",
-    "@types/react": "18.3.12",
-    "@types/react-dom": "18.3.1",
-    "@vitejs/plugin-react": "4.3.4",
-    "@vitest/coverage-v8": "2.1.8",
-    "eslint": "9.16.0",
-    "eslint-plugin-react": "7.37.2",
-    "eslint-plugin-react-hooks": "5.1.0",
-    "eslint-plugin-react-refresh": "0.4.16",
-    "globals": "15.13.0",
-    "typescript": "5.7.2",
-    "typescript-eslint": "8.18.0",
-    "vite": "6.0.3",
-    "vitest": "2.1.8"
   }
 }

apps/hightable-demo/test/package.test.js

@@ -14,7 +14,7 @@ describe('package.json', () => {
   it('should have precise dependency versions', () => {
     const { dependencies, devDependencies } = packageJson
     const allDependencies = { ...dependencies, ...devDependencies }
-    Object.entries(allDependencies).filter(([name]) => !name.startsWith('@hyparam/')).forEach(([, version]) => {
+    Object.values(allDependencies).forEach(version => {
       expect(version).toMatch(/^\d+\.\d+\.\d+$/)
     })
   })

apps/hyparquet-demo/eslint.config.js

@@ -4,8 +4,7 @@ import reactHooks from 'eslint-plugin-react-hooks'
 import reactRefresh from 'eslint-plugin-react-refresh'
 import globals from 'globals'
 import tseslint from 'typescript-eslint'
-import sharedEslintRules from '../../shared.eslint.config.js'
-const { sharedJsRules, sharedTsRules } = sharedEslintRules
+import { sharedJsRules, sharedTsRules } from '../../eslint.config.js'
 
 export default tseslint.config(
   { ignores: ['dist', 'coverage'] },

apps/hyparquet-demo/package.json

@@ -10,7 +10,8 @@
     "build": "tsc -b && vite build",
     "lint": "eslint .",
     "preview": "vite preview",
-    "test": "vitest run --dir test"
+    "test": "vitest run --dir test",
+    "typecheck": "tsc"
   },
   "dependencies": {
     "@hyparam/components": "0.1.11",
@@ -19,21 +20,5 @@
     "hyparquet-compressors": "1.0.0",
     "react": "18.3.1",
     "react-dom": "18.3.1"
-  },
-  "devDependencies": {
-    "@eslint/js": "9.16.0",
-    "@types/react": "18.3.12",
-    "@types/react-dom": "18.3.1",
-    "@vitejs/plugin-react": "4.3.4",
-    "@vitest/coverage-v8": "2.1.8",
-    "eslint": "9.16.0",
-    "eslint-plugin-react": "7.37.2",
-    "eslint-plugin-react-hooks": "5.1.0",
-    "eslint-plugin-react-refresh": "0.4.16",
-    "globals": "15.13.0",
-    "typescript": "5.7.2",
-    "typescript-eslint": "8.18.0",
-    "vite": "6.0.3",
-    "vitest": "2.1.8"
   }
 }

apps/hyparquet-demo/test/package.test.js

@@ -14,7 +14,7 @@ describe('package.json', () => {
   it('should have precise dependency versions', () => {
     const { dependencies, devDependencies } = packageJson
     const allDependencies = { ...dependencies, ...devDependencies }
-    Object.entries(allDependencies).filter(([name]) => !name.startsWith('@hyparam/')).forEach(([, version]) => {
+    Object.values(allDependencies).forEach(version => {
       expect(version).toMatch(/^\d+\.\d+\.\d+$/)
     })
   })