use matrix

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI
+name: CI with dynamic parallel matrix
 
 on:
   workflow_dispatch:
@@ -12,18 +12,20 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  NBCONVERT_JOBS: '4'
-  LITERATE_PROC: '4'
   ALLOWERRORS: 'false'
+  NBCACHE: '.cache'
   CACHE_NUM: '1'
   JULIA_CONDAPKG_BACKEND: 'Null'
+  JULIA_NUM_THREADS: 'auto'
   JULIA_CI: 'true'
-  NBCACHE: '.cache'
   UV_SYSTEM_PYTHON: 1
 
 jobs:
-  CI:
+  setup:
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      hash: ${{ steps.hash.outputs.value }}
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
@@ -32,18 +34,6 @@ jobs:
       id: setup-python
       with:
         python-version: '3.x'
-    - name: Install uv
-      run: curl -LsSf https://astral.sh/uv/install.sh | sh
-    - name: Install Python dependencies
-      run: uv pip install -r requirements.txt
-    - name: Cache executed notebooks
-      uses: actions/cache@v4
-      id: cache-nb
-      with:
-        path: ${{ env.NBCACHE }}
-        key: ${{ runner.os }}-nb-${{ env.CACHE_NUM }}-${{ hashFiles('src/**', 'Project.toml', 'Manifest.toml', 'requirements.txt') }}-${{ hashFiles('docs/**/*.ipynb', 'docs/**/*.jl') }}
-        restore-keys: |
-          ${{ runner.os }}-nb-${{ env.CACHE_NUM }}-${{ hashFiles('src/**', 'Project.toml', 'Manifest.toml', 'requirements.txt') }}-
     - name: Read Julia version
       uses: SebRollen/[email protected]
       id: read_toml
@@ -54,15 +44,18 @@ jobs:
       uses: julia-actions/setup-julia@v2
       with:
         version: ${{ steps.read_toml.outputs.value }}
-    - name: Restore Julia packages
-      uses: actions/cache/restore@v4
+    - name: Get environment hash
+      id: hash
+      run: echo "value=${{ hashFiles('Project.toml', 'Manifest.toml', 'src/**') }}" >> "$GITHUB_OUTPUT"
+    - name: Cache Julia packages
       if: ${{ contains(runner.name, 'GitHub Actions') }}
+      uses: actions/cache@v4
       id: cache-julia
       with:
         path: |
           ~/.julia
           !~/.julia/registries
-        key: ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-${{ steps.read_toml.outputs.value }}-${{ hashFiles('src/**', 'Project.toml', 'Manifest.toml') }}
+        key: ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-${{ steps.read_toml.outputs.value }}-${{ steps.hash.outputs.value }}
         restore-keys: |
           ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-${{ steps.read_toml.outputs.value }}-
     - name: Install Julia packages
@@ -74,34 +67,140 @@ jobs:
         Pkg.activate(".")
         Pkg.instantiate()
         Pkg.precompile()
-        Pkg.gc(collect_delay=Day(0))
-    - name: Save Julia packages
-      uses: actions/cache/save@v4
-      if: ${{ contains(runner.name, 'GitHub Actions') && steps.cache-julia.outputs.cache-hit != 'true' }}
-      with:
-        path: |
-          ~/.julia
-          !~/.julia/registries
-        key: ${{ steps.cache-julia.outputs.cache-primary-key }}
-    - name: Run notebooks
-      run: julia --color=yes -p ${{ env.LITERATE_PROC }} --heap-size-hint=3G run.jl
-    - name: Copy back built notebooks
-      run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
-    - name: Build website
-      run: jupyter-book build docs/
-    - name: Upload pages artifact
-      if: ${{ github.ref == 'refs/heads/main' }}
-      uses: actions/upload-pages-artifact@v3
-      with:
-        path: docs/_build/html/
+    - name: List notebooks as a JSON array
+      id: set-matrix
+      run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs",recursive=True)))')" >> "$GITHUB_OUTPUT"
+
+  execute:
+    needs: setup
+    strategy:
+      max-parallel: 20
+      fail-fast: false
+      matrix:
+        # Notebooks need to be executed
+        notebook: ${{ fromJSON(needs.setup.outputs.matrix) }}
+    runs-on: ubuntu-latest
+    env:
+      NB: docs/${{ matrix.notebook }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Restore notebook if present
+        uses: actions/cache/restore@v4
+        id: nb-cache
+        with:
+          path: ${{ env.NBCACHE }}
+          key: ${{ runner.os }}-notebook-${{ hashFiles(env.NB) }}-${{ needs.setup.outputs.hash }}
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
+        with:
+          python-version: '3.x'
+      - name: Install uv
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install Python dependencies
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
+        run: uv pip install -r requirements.txt
+      - name: Read Julia version
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
+        uses: SebRollen/[email protected]
+        id: read_toml
+        with:
+          file: 'Manifest.toml'
+          field: 'julia_version'
+      - name: Setup Julia
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ steps.read_toml.outputs.value }}
+      - name: Restore Julia packages
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' && contains(runner.name, 'GitHub Actions')}}
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.julia
+            !~/.julia/registries
+          key: ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-${{ steps.read_toml.outputs.value }}-${{ needs.setup.outputs.hash }}
+          restore-keys: |
+            ${{ runner.os }}-julia-${{ env.CACHE_NUM }}-${{ steps.read_toml.outputs.value }}-
+      - name: Execute notebook
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
+        run: julia execute.jl
+      - name: Cache notebook
+        uses: actions/cache/save@v4
+        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
+        with:
+          path: ${{ env.NBCACHE }}
+          key: ${{ steps.nb-cache.outputs.cache-primary-key }}
+      - name: Convert artifact Name
+        id: art
+        run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
+      - name: Upload Notebook
+        uses: actions/upload-artifact@v4
+        with:
+          name: notebook-${{ steps.art.outputs.name }}
+          path: ${{ env.NBCACHE }}
+          retention-days: 1
+
+  render:
+    needs: execute
+    runs-on: ubuntu-latest
+    env:
+      UV_SYSTEM_PYTHON: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Download notebooks
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ env.NBCACHE }}/
+          pattern: notebook-*
+          merge-multiple: true
+      - name: Display structure of downloaded files
+        run: ls -R ${{ env.NBCACHE }}
+      - name: Copy back built notebooks
+        run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install Python dependencies
+        run: uv pip install -r requirements-jb.txt
+      - name: Build website
+        run: jupyter-book build docs/
+      - name: Upload pages artifact
+        if: ${{ github.ref == 'refs/heads/main' }}
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/_build/html/
+
+  # CI conclusion for GitHub status check
+  # Adaped from https://brunoscheufler.com/blog/2022-04-09-the-required-github-status-check-that-wasnt
+  CI:
+    needs: render
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          if [[ ${{ needs.render.result }} == "success" ]]; then
+            echo "Tests passed"
+            exit 0
+          else
+            echo "Tests failed"
+            exit 1
+          fi
 
-  # Deploy pages
+  # Deployment job
   deploy:
-    needs: CI
+    name: Deploy to GitHub pages
+    needs: render
     if: ${{ github.ref == 'refs/heads/main' }}
     # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
     permissions:
-      pages: write # to deploy to Pages
+      pages: write    # to deploy to Pages
       id-token: write # to verify the deployment originates from an appropriate source
       actions: read   # to download an artifact uploaded by `actions/upload-pages-artifact@v3`
     environment:

.github/workflows/linkcheck.yml

@@ -49,6 +49,6 @@ jobs:
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Install Python dependencies
-        run: uv pip install -r requirements.txt
+        run: uv pip install -r requirements-jb.txt
       - name: Build website
         run: jupyter-book build ${DIR} --builder linkcheck

execute.jl

@@ -0,0 +1,64 @@
+using Literate
+using JSON
+using Pkg
+using IJulia
+Pkg.activate(Base.current_project())
+
+ENV["GKSwstype"] = "100"
+
+function main(; rmsvg=true)
+    file = get(ENV, "NB", "test.ipynb")
+    if endswith(file, ".jl")
+        run_literate(file; rmsvg)
+    elseif endswith(file, ".ipynb")
+        IJulia.installkernel("Julia", "--project=@.")
+        run_ipynb(file)
+    else
+        error("$(file) is not a valid notebook file!")
+    end
+end
+
+# Strip SVG output from a Jupyter notebook
+function strip_svg(ipynb)
+    @info "Stripping SVG in $(ipynb)"
+    nb = open(JSON.parse, ipynb, "r")
+    for cell in nb["cells"]
+        !haskey(cell, "outputs") && continue
+        for output in cell["outputs"]
+            !haskey(output, "data") && continue
+            datadict = output["data"]
+            if haskey(datadict, "image/png") || haskey(datadict, "image/jpeg")
+                delete!(datadict, "text/html")
+                delete!(datadict, "image/svg+xml")
+            end
+        end
+    end
+    rm(ipynb)
+    open(ipynb, "w") do io
+        JSON.print(io, nb, 1)
+    end
+    return ipynb
+end
+
+function run_literate(file; rmsvg=true)
+    cachedir = get(ENV, "NBCACHE", ".cache")
+    outpath = joinpath(abspath(pwd()), cachedir, dirname(file))
+    mkpath(outpath)
+    ipynb = Literate.notebook(file, outpath; mdstrings=true, execute=true)
+    rmsvg && strip_svg(ipynb)
+    return
+end
+
+function run_ipynb(file)
+    cachedir = get(ENV, "NBCACHE", ".cache")
+    outpath = joinpath(abspath(pwd()), cachedir, file)
+    mkpath(dirname(outpath))
+    kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)"
+    execute = get(ENV, "ALLOWERRORS", " ") == "true" ? "--execute --allow-errors" : "--execute"
+    timeout = "--ExecutePreprocessor.timeout=" * get(ENV, "TIMEOUT", "-1")
+    cmd = `jupyter nbconvert --to notebook $(execute) $(timeout) $(kernelname) --output $(outpath) $(file)`
+    run(cmd)
+    return
+end
+
+main()

requirements-jb.txt

@@ -0,0 +1 @@
+jupyter-book==1.0.2

requirements.txt

@@ -1,3 +1,2 @@
+# matplotlib==3.9.2
 nbconvert==7.16.4
-jupyter-book==1.0.2
-# matplotlib==3.8.4