From 209992fa30a21acbe9761e31ec325c199d2755d8 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 14 Jul 2022 02:38:21 +0200
Subject: [PATCH 1/4] nodejs builder: process package.json in-place

no need for package-json.bak, just put the original data aside in the JSON.
---
 src/subsystems/nodejs/builders/granular/default.nix |  1 -
 .../nodejs/builders/granular/fix-package.py         | 13 +++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index 7e8c976087..035fea09eb 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -368,7 +368,6 @@
           rm $nodeModules/$packageName/package.json.old
 
           # run python script (see comment above):
-          cp package.json package.json.bak
           python $fixPackage \
           || \
           # exit code 3 -> the package is incompatible to the current platform
diff --git a/src/subsystems/nodejs/builders/granular/fix-package.py b/src/subsystems/nodejs/builders/granular/fix-package.py
index 9c60b4b06f..d67c5e4c1c 100644
--- a/src/subsystems/nodejs/builders/granular/fix-package.py
+++ b/src/subsystems/nodejs/builders/granular/fix-package.py
@@ -14,7 +14,7 @@
 
 changed = False
 
-# fail if platform incompatible
+# fail if platform incompatible - should not happen due to filters
 if 'os' in package_json:
   platform = sys.platform
   if platform not in package_json['os']\
@@ -39,7 +39,7 @@
     f"{package_json.get('version')} -> {version}",
     file=sys.stderr
   )
-  changed = True
+  package_json['origVersion'] = package_json['version'] 
   package_json['version'] = version
 
 
@@ -48,6 +48,7 @@
 # as NPM install will otherwise re-fetch these
 if 'dependencies' in package_json:
   dependencies = package_json['dependencies']
+  depsChanged = False
   # dependencies can be a list or dict
   for pname in dependencies:
     if 'bundledDependencies' in package_json\
@@ -58,17 +59,21 @@
         f"WARNING: Dependency {pname} wanted but not available. Ignoring.",
         file=sys.stderr
       )
+      depsChanged = True
       continue
     version =\
       'unknown' if isinstance(dependencies, list) else dependencies[pname]
     if available_deps[pname] != version:
-      version = available_deps[pname]
-      changed = True
+      depsChanged = True
       print(
         f"package.json: Pinning version '{version}' to '{available_deps[pname]}'"
         f" for dependency '{pname}'",
         file=sys.stderr
       )
+  if depsChanged:
+    changed = True
+    package_json['dependencies'] = available_deps
+    package_json['origDependencies'] = dependencies
 
 # write changes to package.json
 if changed:

From 27a80e5a0593cbaa8cef8f01695ae1045a7d6b65 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 14 Jul 2022 19:30:27 +0200
Subject: [PATCH 2/4] dream-lock: change cyclic deps api

make it easier to implement cycle management
---
 src/default.nix                               |  2 +-
 .../haskell/builders/default/default.nix      | 21 ++++--
 .../nodejs/builders/granular/default.nix      | 24 ++++---
 .../builders/build-rust-package/default.nix   |  2 +-
 .../rust/builders/crane/default.nix           |  5 +-
 src/templates/builders/default.nix            | 21 ++++--
 src/utils/dream-lock.nix                      | 64 ++++++++++++++++++-
 7 files changed, 110 insertions(+), 29 deletions(-)

diff --git a/src/default.nix b/src/default.nix
index 50457e277f..11a365107c 100644
--- a/src/default.nix
+++ b/src/default.nix
@@ -284,7 +284,7 @@ in let
           getSourceSpec
           getRoot
           getDependencies
-          getCyclicDependencies
+          getCyclicHelpers
           defaultPackageName
           defaultPackageVersion
           packages
diff --git a/src/subsystems/haskell/builders/default/default.nix b/src/subsystems/haskell/builders/default/default.nix
index ef09ed63c2..a843f8b778 100644
--- a/src/subsystems/haskell/builders/default/default.nix
+++ b/src/subsystems/haskell/builders/default/default.nix
@@ -14,15 +14,22 @@ in {
   }: {
     ### FUNCTIONS
     # AttrSet -> Bool) -> AttrSet -> [x]
-    getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
-    getDependencies, # name: version: -> [ {name=; version=; } ]
-    getSource, # name: version: -> store-path
+    # name: version: -> helpers
+    getCyclicHelpers,
+    # name: version: -> [ {name=; version=; } ]
+    getDependencies,
+    # name: version: -> store-path
+    getSource,
     # to get information about the original source spec
-    getSourceSpec, # name: version: -> {type="git"; url=""; hash="";}
+    # name: version: -> {type="git"; url=""; hash="";}
+    getSourceSpec,
     ### ATTRIBUTES
-    subsystemAttrs, # attrset
-    defaultPackageName, # string
-    defaultPackageVersion, # string
+    # attrset
+    subsystemAttrs,
+    # string
+    defaultPackageName,
+    # string
+    defaultPackageVersion,
     # all exported (top-level) package names and versions
     # attrset of pname -> version,
     packages,
diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index 035fea09eb..1b315e0ae4 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -14,15 +14,23 @@
     ...
   }: {
     # Funcs
-    # AttrSet -> Bool) -> AttrSet -> [x]
-    getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
-    getDependencies, # name: version: -> [ {name=; version=; } ]
-    getSource, # name: version: -> store-path
+    # name: version: -> helpers
+    getCyclicHelpers,
+    # name: version: -> [ {name=; version=; } ]
+    getDependencies,
+    # name: version: -> store-path
+    getSource,
+    # name: version: -> {type="git"; url=""; hash="";} + extra values from npm packages
+    getSourceSpec,
     # Attributes
-    subsystemAttrs, # attrset
-    defaultPackageName, # string
-    defaultPackageVersion, # string
-    packages, # list
+    # attrset
+    subsystemAttrs,
+    # string
+    defaultPackageName,
+    # string
+    defaultPackageVersion,
+    # list
+    packages,
     # attrset of pname -> versions,
     # where versions is a list of version strings
     packageVersions,
diff --git a/src/subsystems/rust/builders/build-rust-package/default.nix b/src/subsystems/rust/builders/build-rust-package/default.nix
index 7ad90c891d..6b2a6ff278 100644
--- a/src/subsystems/rust/builders/build-rust-package/default.nix
+++ b/src/subsystems/rust/builders/build-rust-package/default.nix
@@ -9,7 +9,7 @@
     subsystemAttrs,
     defaultPackageName,
     defaultPackageVersion,
-    getCyclicDependencies,
+    getCyclicHelpers,
     getDependencies,
     getSource,
     getSourceSpec,
diff --git a/src/subsystems/rust/builders/crane/default.nix b/src/subsystems/rust/builders/crane/default.nix
index a708c84c0c..cfd577dd99 100644
--- a/src/subsystems/rust/builders/crane/default.nix
+++ b/src/subsystems/rust/builders/crane/default.nix
@@ -9,7 +9,7 @@
     subsystemAttrs,
     defaultPackageName,
     defaultPackageVersion,
-    getCyclicDependencies,
+    getCyclicHelpers,
     getDependencies,
     getSource,
     getSourceSpec,
@@ -27,7 +27,8 @@
       then externals.crane toolchain
       else if toolchain ? cargo
       then
-        externals.crane {
+        externals.crane
+        {
           cargoHostTarget = toolchain.cargo;
           cargoBuildBuild = toolchain.cargo;
         }
diff --git a/src/templates/builders/default.nix b/src/templates/builders/default.nix
index cadb984f86..befd3c01e1 100644
--- a/src/templates/builders/default.nix
+++ b/src/templates/builders/default.nix
@@ -11,15 +11,22 @@
   }: {
     ### FUNCTIONS
     # AttrSet -> Bool) -> AttrSet -> [x]
-    getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
-    getDependencies, # name: version: -> [ {name=; version=; } ]
-    getSource, # name: version: -> store-path
+    # name: version: -> helpers
+    getCyclicHelpers,
+    # name: version: -> [ {name=; version=; } ]
+    getDependencies,
+    # name: version: -> store-path
+    getSource,
     # to get information about the original source spec
-    getSourceSpec, # name: version: -> {type="git"; url=""; hash="";}
+    # name: version: -> {type="git"; url=""; hash="";}
+    getSourceSpec,
     ### ATTRIBUTES
-    subsystemAttrs, # attrset
-    defaultPackageName, # string
-    defaultPackageVersion, # string
+    # attrset
+    subsystemAttrs,
+    # string
+    defaultPackageName,
+    # string
+    defaultPackageVersion,
     # all exported (top-level) package names and versions
     # attrset of pname -> version,
     packages,
diff --git a/src/utils/dream-lock.nix b/src/utils/dream-lock.nix
index e766527fb7..dad20634b2 100644
--- a/src/utils/dream-lock.nix
+++ b/src/utils/dream-lock.nix
@@ -136,9 +136,67 @@
       (dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or [])
       dependencyGraph."${pname}"."${version}" or [];
 
-    getCyclicDependencies = pname: version:
-      cyclicDependencies."${pname}"."${version}" or [];
+    # inverted cyclicDependencies { name.version = parent }
+    cyclicParents = with l;
+      foldAttrs (c: acc: acc // (listToAttrs [(nameValuePair c.version c.cyclic)])) {} (flatten (mapAttrsToList (cyclicName: cyclicVersions:
+        mapAttrsToList (cyclicVersion: cycleeDeps:
+          map (cycleeDep: (listToAttrs [
+            (
+              nameValuePair cycleeDep.name
+              {
+                version = cycleeDep.version;
+                cyclic = {
+                  name = cyclicName;
+                  version = cyclicVersion;
+                };
+              }
+            )
+          ]))
+          cycleeDeps)
+        cyclicVersions)
+      cyclicDependencies));
+
+    getCyclicHelpers = name: version: let
+      # [ {name; version} ]
+      cycleeDeps = cyclicDependencies."${name}"."${version}" or [];
+
+      # {name: {version: true}}
+      cycleeMap = lib.foldAttrs (depVersion: acc:
+        acc
+        // (lib.listToAttrs [
+          {
+            name = depVersion;
+            value = true;
+          }
+        ])) {}
+      cycleeDeps;
+
+      cyclicParent = cyclicParents."${name}"."${version}" or null;
+      isCyclee = depName: depVersion: cycleeMap."${depName}"."${depVersion}" or false;
+      isThisCycleeFor = depName: depVersion:
+        cyclicParent
+        == {
+          name = depName;
+          version = depVersion;
+        };
 
+      replaceCyclees = deps:
+        with l;
+          filter (d: d != null)
+          (map (d: let
+            parent = cyclicParents."${d.name}"."${d.version}" or null;
+          in
+            if parent != null
+            then
+              if parent == cyclicParent
+              # These packages will be part of their parent package
+              then null
+              else parent // {replaces = d;}
+            else d)
+          deps);
+    in {
+      inherit cycleeDeps cyclicParent isCyclee isThisCycleeFor replaceCyclees;
+    };
     getRoot = pname: version: let
       spec = getSourceSpec pname version;
     in
@@ -157,7 +215,7 @@
         defaultPackageName
         defaultPackageVersion
         subsystemAttrs
-        getCyclicDependencies
+        getCyclicHelpers
         getDependencies
         getSourceSpec
         getRoot

From c8b37592336eb57422223cd6b073fd28089996b7 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 11 Aug 2022 14:36:02 +0200
Subject: [PATCH 3/4] nodejs builder: some script optimizations

- prevent bad permissions if possible
- fix bad permissions with single chmod command
- join multiple jq calls
- make electron wrapping more readable
---
 .../nodejs/builders/granular/default.nix      | 66 ++++++-------------
 1 file changed, 21 insertions(+), 45 deletions(-)

diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index 1b315e0ae4..587e7ec4e6 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -112,7 +112,7 @@
       local ver
       ver="v$(cat $electronDist/version | tr -d '\n')"
       mkdir $TMP/$ver
-      cp $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz
+      cp --no-preserve=mode $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz
 
       # calc checksums
       cd $TMP/$ver
@@ -123,18 +123,11 @@
       python -m http.server 45034 --directory $TMP &
 
       # copy electron distribution
-      cp -r $electronDist $TMP/electron
-      chmod -R +w $TMP/electron
+      cp -r --no-preserve=mode $electronDist $TMP/electron
 
       # configure electron toolchain
-      ${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \
-          | ${pkgs.moreutils}/bin/sponge package.json
-
-      ${pkgs.jq}/bin/jq ".build.linux.target = \"dir\"" package.json \
-          | ${pkgs.moreutils}/bin/sponge package.json
-
-      ${pkgs.jq}/bin/jq ".build.npmRebuild = false" package.json \
-          | ${pkgs.moreutils}/bin/sponge package.json
+      jq ".build.electronDist = \"$TMP/electron\" | .build.linux.target = \"dir\" | .build.npmRebuild = false" package.json > package.json.tmp
+      mv package.json.tmp package.json
 
       # execute electron-rebuild if available
       export headers=http://localhost:45034/
@@ -148,22 +141,10 @@
 
     # Only executed for electron based packages.
     # Creates an executable script under /bin starting the electron app
-    electron-wrap =
+    electronBin =
       if pkgs.stdenv.isLinux
-      then ''
-        mkdir -p $out/bin
-        makeWrapper \
-          $electronDist/electron \
-          $out/bin/$(basename "$packageName") \
-          --add-flags "$(realpath $electronAppDir)"
-      ''
-      else ''
-        mkdir -p $out/bin
-        makeWrapper \
-          $electronDist/Electron.app/Contents/MacOS/Electron \
-          $out/bin/$(basename "$packageName") \
-          --add-flags "$(realpath $electronAppDir)"
-      '';
+      then "$electronDist/electron"
+      else "$electronDist/Electron.app/Contents/MacOS/Electron";
 
     # Generates a derivation for a specific package name + version
     makePackage = name: version: let
@@ -290,25 +271,17 @@
                 continue
               fi
               echo "copying $f"
-              chmod +wx $(dirname "$f")
-              mv "$f" "$f.bak"
-              mkdir "$f"
-              if [ -n "$(ls -A "$f.bak/")" ]; then
-                cp -r "$f.bak"/* "$f/"
-                chmod -R +w $f
-              fi
-              rm "$f.bak"
+              l=$(readlink -f $f)
+              rm -f "$f"
+              cp -r --no-preserve=mode "$l" "$f"
             done
           }
         '';
 
-        # TODO: upstream fix to nixpkgs
+        # https://github.com/NixOS/nixpkgs/pull/50961#issuecomment-449638192
         # example which requires this:
         #   https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz
-        unpackCmd =
-          if lib.hasSuffix ".tgz" src
-          then "tar --delay-directory-restore -xf $src"
-          else null;
+        TAR_OPTIONS = "--delay-directory-restore";
 
         unpackPhase = ''
           runHook preUnpack
@@ -336,9 +309,8 @@
               # Figure out what directory has been unpacked
               export packageDir="$(find . -maxdepth 1 -type d | tail -1)"
 
-              # Restore write permissions
-              find "$packageDir" -type d -exec chmod u+x {} \;
-              chmod -R u+w -- "$packageDir"
+              # Ensure write + directory execute permissions
+              chmod -R u+w,a+X -- "$packageDir"
 
               # Move the extracted tarball into the output folder
               mv -- "$packageDir" "$sourceRoot"
@@ -346,8 +318,8 @@
           then
               export strippedName="$(stripHash $src)"
 
-              # Restore write permissions
-              chmod -R u+w -- "$strippedName"
+              # Ensure write + directory execute permissions
+              chmod -R u+w,a+X -- "$strippedName"
 
               # Move the extracted directory into the output folder
               mv -- "$strippedName" "$sourceRoot"
@@ -495,7 +467,11 @@
           # wrap electron app
           if [ -n "$electronHeaders" ]; then
             echo "Wrapping electron app"
-            ${electron-wrap}
+            mkdir -p $out/bin
+            makeWrapper \
+              ${electronBin} \
+              $out/bin/$(basename "$packageName") \
+              --add-flags "$(realpath $electronAppDir)"
           fi
 
           runHook postInstall

From 3a170a110c63b2e71a219a6512086fb223dcc576 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 11 Aug 2022 14:36:02 +0200
Subject: [PATCH 4/4] nodejs builder: implement tree-of-symlinks

- works out-of-the-box, no node|tsc settings necessary
- optimal use of storage, composable
- handles cyclic dependencies by co-locating cycles in the same store path

Changes:
- build node_modules as a separate derivation, use everywhere
- if main package has a build script, run it with dev modules and
  afterwards run install script if present, then switch to prod modules
- only run build scripts when necessary, speedup
- optionally add all transitive binaries to the node_modules/.bin
- store binaries in bin/
  NixOS standard location, and .bin should only be used for dependencies,
  not the main package
- in a package, .bin is now a symlink to bin
- in bin name, strip .js ending for string case
- remove now-unnecessary code

This builder requires all peer dependencies for packages to be mentioned and all transitive circular dependencies to be grouped
---
 .../nodejs/builders/granular/default.nix      | 399 +++++++++++++-----
 .../nodejs/builders/granular/devShell.nix     |  62 +--
 .../nodejs/builders/granular/install-deps.py  | 212 ----------
 .../nodejs/builders/granular/link-bins.py     |   2 -
 .../builders/granular/tsconfig-to-json.js     |  19 -
 5 files changed, 301 insertions(+), 393 deletions(-)
 delete mode 100644 src/subsystems/nodejs/builders/granular/install-deps.py
 delete mode 100644 src/subsystems/nodejs/builders/granular/tsconfig-to-json.js

diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index 587e7ec4e6..2a9b50faa6 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -45,22 +45,18 @@
     b = builtins;
     l = lib // builtins;
 
-    nodejsVersion = subsystemAttrs.nodejsVersion;
+    nodejsVersion = subsystemAttrs.nodejsVersion or null;
+    transitiveBinaries = subsystemAttrs.transitiveBinaries or false;
 
     isMainPackage = name: version:
       (args.packages."${name}" or null) == version;
 
     nodejs =
-      if args ? nodejs
-      then args.nodejs
-      else
+      args.nodejs
+      or (
         pkgs."nodejs-${builtins.toString nodejsVersion}_x"
-        or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
-
-    nodeSources = runCommandLocal "node-sources" {} ''
-      tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
-      mv node-* $out
-    '';
+        or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs")
+      );
 
     allPackages =
       lib.mapAttrs
@@ -148,14 +144,139 @@
 
     # Generates a derivation for a specific package name + version
     makePackage = name: version: let
-      pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name;
-
-      deps = getDependencies name version;
-
-      nodeDeps =
-        lib.forEach
-        deps
-        (dep: allPackages."${dep.name}"."${dep.version}");
+      pname = name;
+
+      rawDeps = getDependencies name version;
+      inherit (getCyclicHelpers name version) cycleeDeps cyclicParent isCyclee isThisCycleeFor replaceCyclees;
+
+      # cycles
+      # for nodejs, we need to copy any cycles into a single package together
+      # getCyclicHelpers already cut the cycles for us, into one cyclic (e.g. eslint) and many cyclee (e.g. eslint-util)
+      # when a package is cyclic:
+      # - the cyclee deps should not be in the cyclic/node_modules folder
+      # - the cyclee deps need to be copied into the package next to cyclic
+      #   so node can find them all together
+      # when a package is cyclee:
+      # - the cyclic dep should not be in the cyclee/node_modules folder
+      # when a dep is cyclee:
+      # - the dep path should point into the cyclic parent
+
+      # Keep only the deps we can install, assume it all works out
+      deps = let
+        myOS = with stdenv.targetPlatform;
+          if isLinux
+          then "linux"
+          else if isDarwin
+          then "darwin"
+          else "";
+      in
+        replaceCyclees (lib.filter
+          (
+            dep: let
+              p = allPackages."${dep.name}"."${dep.version}";
+              s = p.extraInfo;
+            in
+              # this dep is a cyclee
+              !(isCyclee dep.name dep.version)
+              # this dep is not for this os
+              && ((s.os or null == null) || lib.any (o: o == myOS) s.os)
+              # this package is a cyclee
+              && !(isThisCycleeFor dep.name dep.version)
+          )
+          rawDeps);
+
+      nodePkgs =
+        l.map
+        (dep: let
+          pkg = allPackages."${dep.name}"."${dep.version}";
+        in
+          if dep ? replaces
+          then pkg // {packageName = dep.replaces.name;}
+          else pkg)
+        deps;
+      cycleePkgs =
+        l.map
+        (dep: allPackages."${dep.name}"."${dep.version}")
+        cycleeDeps;
+
+      # Derivation building the ./node_modules directory in isolation.
+      makeModules = {
+        withDev ? false,
+        withOptionals ? true,
+      }: let
+        isMain = isMainPackage name version;
+        # These flags will only be present if true. Also, dev deps are required for non-main packages
+        myDeps =
+          lib.filter
+          (dep: let
+            s = dep.extraInfo;
+          in
+            (withOptionals || !(s.optional or false))
+            && (!isMain || (withDev || !(s.dev or false))))
+          nodePkgs;
+      in
+        if lib.length myDeps == 0
+        then null
+        else
+          pkgs.runCommandLocal "node_modules-${pname}" {} ''
+            shopt -s nullglob
+            set -e
+
+            mkdir $out
+
+            function doLink() {
+              local name=$(basename $1)
+              local target="$2/$name"
+              if [ -e "$target" ]; then
+                local link=$(readlink $target)
+                if [ "$link" = $1 ]; then
+                  # cyclic dep, all ok
+                  return
+                fi
+                echo "Cannot overwrite $target (-> $link) with $1 - incorrect cycle! Versions issue?" >&2
+                exit 1
+              fi
+              ln -s $1 $target
+            }
+
+            function linkDep() {
+              local pkg=$1
+              local name=$2
+              # special case for namespaced modules
+              if [[ $name == @* ]]; then
+                local namespace=$(dirname $name)
+                mkdir -p $out/$namespace
+                doLink $pkg/lib/node_modules/$name $out/$namespace
+              else
+                doLink $pkg/lib/node_modules/$name $out
+              fi
+            }
+
+            ${l.toString (l.map
+              (d: "linkDep ${l.toString d} ${d.packageName}\n")
+              myDeps)}
+
+            # symlink module executables to ./node_modules/.bin
+            mkdir $out/.bin
+            for dep in ${l.toString myDeps}; do
+              # We assume dotfiles are not public binaries
+              for b in $dep/bin/*; do
+                if [ -L "$b" ]; then
+                  # when these relative symlinks, make absolute
+                  # last one wins (-sf)
+                  ln -sf $(readlink -f $b) $out/.bin/$(basename $b)
+                else
+                  # e.g. wrapped binary
+                  ln -sf $b $out/.bin/$(basename $b)
+                fi
+              done
+            done
+            # remove empty .bin
+            rmdir $out/.bin || true
+          '';
+      prodModules = makeModules {withDev = false;};
+      # if noDev was used, these are just the prod modules
+      devModules = makeModules {withDev = true;};
 
       passthruDeps =
         l.listToAttrs
@@ -192,13 +313,22 @@
         then null
         else pkgs."electron_${electronVersionMajor}".headers;
 
+      hasExtraInfo = subsystemAttrs ? extraInfo;
+      extraInfo = subsystemAttrs.extraInfo.${name}.${version} or {};
+      # If the translator doesn't provide extraInfo, assume scripts
+      hasInstall =
+        if hasExtraInfo
+        then extraInfo.hasInstallScript or false
+        else true;
+      isMain = isMainPackage name version;
+
       pkg = produceDerivation name (stdenv.mkDerivation rec {
         inherit
           dependenciesJson
           electronHeaders
-          nodeDeps
-          nodeSources
           version
+          transitiveBinaries
+          prodModules
           ;
 
         packageName = name;
@@ -208,52 +338,63 @@
         passthru.dependencies = passthruDeps;
 
         passthru.devShell = import ./devShell.nix {
-          inherit
-            mkShell
-            nodejs
-            packageName
-            pkg
-            ;
+          inherit mkShell nodejs devModules;
         };
 
+        passthru.extraInfo = extraInfo;
+
         /*
         For top-level packages install dependencies as full copies, as this
         reduces errors with build tooling that doesn't cope well with
         symlinking.
         */
-        installMethod =
-          if isMainPackage name version
-          then "copy"
-          else "symlink";
+        # TODO implement copy and make configurable
+        # installMethod =
+        #   if isMain
+        #   then "copy"
+        #   else "symlink";
 
         electronAppDir = ".";
 
         # only run build on the main package
-        runBuild = isMainPackage name version;
+        runBuild = isMain && (subsystemAttrs.hasBuildScript or true);
 
-        src = getSource name version;
-
-        nativeBuildInputs = [makeWrapper];
-
-        buildInputs = [jq nodejs python3];
+        # can be overridden to define alternative install command
+        # (defaults to npm install steps)
+        buildScript = null;
+        shouldBuild = hasInstall || runBuild || buildScript != null || electronHeaders != null;
+        buildModules =
+          if runBuild
+          then devModules
+          else prodModules;
+        nodeSources =
+          if shouldBuild
+          then nodejs
+          else null;
+
+        # We don't need unpacked sources
+        src = let t = getSource name version; in t.original or t;
+
+        nativeBuildInputs =
+          if shouldBuild
+          then [makeWrapper]
+          else [];
+
+        # We must provide nodejs even when not building to allow
+        # patchShebangs to find it for binaries
+        buildInputs =
+          if shouldBuild || (!hasExtraInfo || (extraInfo ? bin))
+          then [jq nodejs python3]
+          else [python3];
 
         # prevents running into ulimits
-        passAsFile = ["dependenciesJson" "nodeDeps"];
+        passAsFile = ["dependenciesJson"];
 
         preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];
 
-        # can be overridden to define alternative install command
-        # (defaults to 'npm run postinstall')
-        buildScript = null;
-
         # python script to modify some metadata to support installation
         # (see comments below on d2nPatchPhase)
         fixPackage = "${./fix-package.py}";
-
-        # script to install (symlink or copy) dependencies.
-        installDeps = "${./install-deps.py}";
-
-        # python script to link bin entries from package.json
         linkBins = "${./link-bins.py}";
 
         # costs performance and doesn't seem beneficial in most scenarios
@@ -325,28 +466,30 @@
               mv -- "$strippedName" "$sourceRoot"
           fi
 
+          # provide bin, we'll remove it if unused
+          mkdir $out/bin
+          # We keep the binaries in /bin but node uses .bin
+          # Symlink so that wrapper scripts etc work
+          ln -s ../../bin $nodeModules/.bin
+
           runHook postUnpack
         '';
 
         # The python script wich is executed in this phase:
         #   - ensures that the package is compatible to the current system
+        #     (if not already filtered above with os prop from translator)
         #   - ensures the main version in package.json matches the expected
         #   - pins dependency versions in package.json
         #     (some npm commands might otherwise trigger networking)
         #   - creates symlinks for executables declared in package.json
+        #   - Any usage of 'link:' in deps will be replaced with the exact version
         # Apart from that:
-        #   - Any usage of 'link:' in package.json is replaced with 'file:'
         #   - If package-lock.json exists, it is deleted, as it might conflict
         #     with the parent package-lock.json.
         d2nPatchPhase = ''
           # delete package-lock.json as it can lead to conflicts
           rm -f package-lock.json
 
-          # repair 'link:' -> 'file:'
-          mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old
-          cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
-          rm $nodeModules/$packageName/package.json.old
-
           # run python script (see comment above):
           python $fixPackage \
           || \
@@ -359,48 +502,44 @@
           else
             exit 1
           fi
-
-          # configure typescript
-          if [ -f ./tsconfig.json ] \
-              && node -e 'require("typescript")' &>/dev/null; then
-            node ${./tsconfig-to-json.js}
-            ${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \
-                | ${pkgs.moreutils}/bin/sponge tsconfig.json
-          fi
         '';
 
-        # - installs dependencies into the node_modules directory
-        # - adds executables of direct node module dependencies to PATH
-        # - adds the current node module to NODE_PATH
+        # - links dependencies into the node_modules directory + adds bin to PATH
         # - sets HOME=$TMPDIR, as this is required by some npm scripts
-        # TODO: don't install dev dependencies. Load into NODE_PATH instead
         configurePhase = ''
           runHook preConfigure
 
-          # symlink sub dependencies as well as this imitates npm better
-          python $installDeps
-
-          echo "Symlinking transitive executables to $nodeModules/.bin"
-          for dep in ${l.toString nodeDeps}; do
-            binDir=$dep/lib/node_modules/.bin
-            if [ -e $binDir ]; then
-              for bin in $(ls $binDir/); do
-                mkdir -p $nodeModules/.bin
-
-                # symlink might have been already created by install-deps.py
-                # if installMethod=copy was selected
-                if [ ! -e $nodeModules/.bin/$bin ]; then
-                  ln -s $binDir/$bin $nodeModules/.bin/$bin
+          if [ -n "$buildModules" ]; then
+            if [ -L $sourceRoot/node_modules ] || [ -e $sourceRoot/node_modules ]; then
+              echo Warning: The source $sourceRoot includes a node_modules directory. Replacing. >&2
+              rm -rf $sourceRoot/node_modules
+            fi
+            ln -s $buildModules $sourceRoot/node_modules
+            if [ -d $buildModules/.bin ]; then
+              export PATH="$PATH:$sourceRoot/node_modules/.bin"
+            fi
+          fi
+          ${
+            # Here we copy cyclee deps into the cyclehead node_modules
+            # so the cyclic deps can find each other
+            if cycleePkgs != []
+            then ''
+              for dep in ${l.toString cycleePkgs}; do
+                # We must copy everything so Node finds it
+                # Let's hope that clashing names are just duplicates
+                # keep write perms with no-preserve
+                cp -rf --no-preserve=mode $dep/lib/node_modules/* $nodeModules
+                if [ -d $dep/bin ]; then
+                  # this copies symlinks as-is, so they will point to the
+                  # local target when relative, and module-local links
+                  # are made relative by nixpkgs post-build
+                  # last one wins (-f)
+                  cp -af --no-preserve=mode $dep/bin/. $out/bin/.
                 fi
               done
-            fi
-          done
-
-          # add bin path entries collected by python script
-          export PATH="$PATH:$nodeModules/.bin"
-
-          # add dependencies to NODE_PATH
-          export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
+            ''
+            else ""
+          }
 
           export HOME=$TMPDIR
 
@@ -409,47 +548,77 @@
 
         # Runs the install command which defaults to 'npm run postinstall'.
         # Allows using custom install command by overriding 'buildScript'.
-        buildPhase = ''
-          runHook preBuild
+        # TODO this logic supposes a build script, which is not documented
+        # for installing, we only need to run `npm run install` (pre and post scripts run automatically)
+        # https://github.com/npm/npm/issues/5919
+        # TODO build first if has build, give it devModules during build
+
+        buildPhase =
+          if shouldBuild
+          then ''
+            set -x
+            runHook preBuild
+
+            if [ -n "$shouldBuild" ]; then
+              # execute electron-rebuild
+              if [ -n "$electronHeaders" ]; then
+                echo "executing electron-rebuild"
+                ${electron-rebuild}
+              fi
 
-          # execute electron-rebuild
-          if [ -n "$electronHeaders" ]; then
-            echo "executing electron-rebuild"
-            ${electron-rebuild}
-          fi
+              # execute install command
+              if [ -n "$buildScript" ]; then
+                if [ -f "$buildScript" ]; then
+                  $buildScript
+                else
+                  eval "$buildScript"
+                fi
+              else
+                if [ -n "$runBuild" ]; then
+                  # by default, only for top level packages, `npm run build` is executed
+                  npm run --if-present build
+                fi
 
-          # execute install command
-          if [ -n "$buildScript" ]; then
-            if [ -f "$buildScript" ]; then
-              $buildScript
-            else
-              eval "$buildScript"
-            fi
-          # by default, only for top level packages, `npm run build` is executed
-          elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
-            npm run build
-          else
-            if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then
-              npm --production --offline --nodedir=$nodeSources run preinstall
-            fi
-            if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
-              npm --production --offline --nodedir=$nodeSources run install
-            fi
-            if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
-              npm --production --offline --nodedir=$nodeSources run postinstall
+                # This seems to be the only script that needs running on install
+                npm --omit=dev --offline --nodedir=$nodeSources run --if-present install
+              fi
             fi
-          fi
 
-          runHook postBuild
-        '';
+            runHook postBuild
+            set +x
+          ''
+          else "true";
 
         # Symlinks executables and manual pages to correct directories
         installPhase = ''
           runHook preInstall
 
+          if [ "$buildModules" != "$prodModules" ]; then
+            if [ -n "$prodModules" ]; then
+              ln -sf $prodModules $sourceRoot/node_modules
+            else
+              rm $sourceRoot/node_modules
+            fi
+          fi
+
           echo "Symlinking bin entries from package.json"
           python $linkBins
 
+          if [ -n "$transitiveBinaries" ]; then
+            # pass down transitive binaries, like npm does
+            # all links are absolute so we can just copy
+            cp -af --no-preserve=mode $prodModules/.bin/. $out/bin/.
+          fi
+
+          if rmdir $out/bin 2>/dev/null; then
+            # we didn't install any binaries
+            rm $nodeModules/.bin
+          else
+            # make sure binaries are executable, following symlinks
+            # ignore failures from symlinks pointing to other pkgs
+            chmod a+x $out/bin/* 2>/dev/null || true
+          fi
+
           echo "Symlinking manual pages"
           if [ -d "$nodeModules/$packageName/man" ]
           then
diff --git a/src/subsystems/nodejs/builders/granular/devShell.nix b/src/subsystems/nodejs/builders/granular/devShell.nix
index 82b69c556b..b1b92e509d 100644
--- a/src/subsystems/nodejs/builders/granular/devShell.nix
+++ b/src/subsystems/nodejs/builders/granular/devShell.nix
@@ -17,51 +17,23 @@ with a fully reproducible copy again.
 {
   mkShell,
   nodejs,
-  packageName,
-  pkg,
+  devModules,
 }:
 mkShell {
-  buildInputs = [
-    nodejs
-  ];
-  shellHook = let
-    /*
-    This uses the existig package derivation, and modifies it, to
-    disable all phases but the one which creates the ./node_modules.
-
-    The result is a derivation only generating the node_modules and
-    .bin directories.
-
-    TODO: This is be a bit hacky and could be abstracted better
-    TODO: Don't always delete all of ./node_modules. Only overwrite
-          missing or changed modules.
-    */
-    nodeModulesDrv = pkg.overrideAttrs (old: {
-      buildPhase = ":";
-      installMethod = "copy";
-      dontPatch = true;
-      dontBuild = true;
-      dontInstall = true;
-      dontFixup = true;
-      # the configurePhase fails if these variables are not set
-      d2nPatchPhase = ''
-        nodeModules=$out/lib/node_modules
-        mkdir -p $nodeModules/$packageName
-        cd $nodeModules/$packageName
-      '';
-    });
-    nodeModulesDir = "${nodeModulesDrv}/lib/node_modules/${packageName}/node_modules";
-    binDir = "${nodeModulesDrv}/lib/node_modules/.bin";
-  in ''
-    # create the ./node_modules directory
-    rm -rf ./node_modules
-    mkdir -p ./node_modules/.bin
-    cp -r ${nodeModulesDir}/* ./node_modules/
-    for link in $(ls ${binDir}); do
-      target=$(readlink ${binDir}/$link | cut -d'/' -f4-)
-      ln -s ../$target ./node_modules/.bin/$link
-    done
-    chmod -R +w ./node_modules
-    export PATH="$PATH:$(realpath ./node_modules)/.bin"
-  '';
+  buildInputs = [nodejs];
+  # TODO implement copy, maybe
+  shellHook =
+    if devModules != null
+    then ''
+      # create the ./node_modules directory
+      if [ -e ./node_modules ] && [ ! -L ./node_modules ]; then
+        echo -e "\nFailed creating the ./node_modules symlink to '${devModules}'"
+        echo -e "\n./node_modules already exists and is a directory, which means it is managed by another program. Please delete ./node_modules first and re-enter the dev shell."
+      else
+        rm -f ./node_modules
+        ln -s ${devModules} ./node_modules
+        export PATH="$PATH:$(realpath ./node_modules)/.bin"
+      fi
+    ''
+    else "";
 }
diff --git a/src/subsystems/nodejs/builders/granular/install-deps.py b/src/subsystems/nodejs/builders/granular/install-deps.py
deleted file mode 100644
index 9d74e09924..0000000000
--- a/src/subsystems/nodejs/builders/granular/install-deps.py
+++ /dev/null
@@ -1,212 +0,0 @@
-import json
-import os
-import pathlib
-import shutil
-import subprocess as sp
-import sys
-
-
-pname = os.environ.get('packageName')
-version = os.environ.get('version')
-bin_dir = f"{os.path.abspath('..')}/.bin"
-root = f"{os.path.abspath('.')}/node_modules"
-package_json_cache = {}
-
-
-with open(os.environ.get("nodeDepsPath")) as f:
-  nodeDeps = f.read().split()
-
-def get_package_json(path):
-  if path not in package_json_cache:
-    if not os.path.isfile(f"{path}/package.json"):
-      return None
-    with open(f"{path}/package.json") as f:
-      package_json_cache[path] = json.load(f)
-  return package_json_cache[path]
-
-def install_direct_dependencies():
-  if not os.path.isdir(root):
-    os.mkdir(root)
-  with open(os.environ.get('nodeDepsPath')) as f:
-    deps = f.read().split()
-  for dep in deps:
-    if os.path.isdir(f"{dep}/lib/node_modules"):
-      for module in os.listdir(f"{dep}/lib/node_modules"):
-        # ignore hidden directories
-        if module[0] == ".":
-          continue
-        if module[0] == '@':
-          for submodule in os.listdir(f"{dep}/lib/node_modules/{module}"):
-            pathlib.Path(f"{root}/{module}").mkdir(exist_ok=True)
-            print(f"installing: {module}/{submodule}")
-            origin =\
-              os.path.realpath(f"{dep}/lib/node_modules/{module}/{submodule}")
-            if not os.path.exists(f"{root}/{module}/{submodule}"):
-              os.symlink(origin, f"{root}/{module}/{submodule}")
-        else:
-          print(f"installing: {module}")
-          origin = os.path.realpath(f"{dep}/lib/node_modules/{module}")
-          if not os.path.isdir(f"{root}/{module}"):
-            os.symlink(origin, f"{root}/{module}")
-          else:
-            print(f"already exists: {root}/{module}")
-
-
-def collect_dependencies(root, depth):
-  if not os.path.isdir(root):
-    return []
-  dirs = os.listdir(root)
-
-  currentDeps = []
-  for d in dirs:
-    if d.rpartition('/')[-1].startswith('@'):
-      subdirs = os.listdir(f"{root}/{d}")
-      for sd in subdirs:
-        cur_dir = f"{root}/{d}/{sd}"
-        currentDeps.append(f"{cur_dir}")
-    else:
-      cur_dir = f"{root}/{d}"
-      currentDeps.append(cur_dir)
-
-  if depth == 0:
-    return currentDeps
-  else:
-    depsOfDeps =\
-      map(lambda dep: collect_dependencies(f"{dep}/node_modules", depth - 1), currentDeps)
-    result = []
-    for deps in depsOfDeps:
-      result += deps
-    return result
-
-
-def symlink_sub_dependencies():
-  for dep in collect_dependencies(root, 1):
-    # compute module path
-    d1, d2 = dep.split('/')[-2:]
-    if d1.startswith('@'):
-      path = f"{root}/{d1}/{d2}"
-    else:
-      path = f"{root}/{d2}"
-
-    # check for collision
-    if os.path.isdir(path):
-      continue
-
-    # create parent dir
-    pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
-
-    # symlink dependency
-    os.symlink(os.path.realpath(dep), path)
-
-
-# create symlinks for executables (bin entries from package.json)
-def symlink_bin(bin_dir, package_location, package_json, force=False):
-  if package_json and 'bin' in package_json and package_json['bin']:
-    bin = package_json['bin']
-
-    def link(name, relpath):
-      source = f'{bin_dir}/{name}'
-      sourceDir = os.path.dirname(source)
-      # create parent dir
-      pathlib.Path(sourceDir).mkdir(parents=True, exist_ok=True)
-      dest = os.path.relpath(f'{package_location}/{relpath}', sourceDir)
-      print(f"symlinking executable. dest: {dest}; source: {source}")
-      if force and os.path.exists(source):
-        os.remove(source)
-      if not os.path.exists(source):
-        os.symlink(dest, source)
-
-    if isinstance(bin, str):
-      name = package_json['name'].split('/')[-1]
-      link(name, bin)
-
-    else:
-      for name, relpath in bin.items():
-        link(name, relpath)
-
-
-# checks if dependency is already installed in the current or parent dir.
-def dependency_satisfied(root, pname, version):
-  if root == "/nix/store":
-    return False
-
-  parent = os.path.dirname(root)
-
-  if os.path.isdir(f"{root}/{pname}"):
-    package_json_file = f"{root}/{pname}/package.json"
-    if os.path.isfile(package_json_file):
-      if version == get_package_json(f"{root}/{pname}").get('version'):
-        return True
-
-  return dependency_satisfied(parent, pname, version)
-
-
-# transforms symlinked dependencies into real copies
-def symlinks_to_copies(node_modules):
-  sp.run(f"chmod +wx {node_modules}".split())
-  for dep in collect_dependencies(node_modules, 0):
-
-    # only handle symlinks to directories
-    if not os.path.islink(dep) or os.path.isfile(dep):
-      continue
-
-    d1, d2 = dep.split('/')[-2:]
-    if d1[0] == '@':
-      pname = f"{d1}/{d2}"
-      sp.run(f"chmod +wx {node_modules}/{d1}".split())
-    else:
-      pname = d2
-
-    package_json = get_package_json(dep)
-    if package_json is not None:
-      version = package_json['version']
-      if dependency_satisfied(os.path.dirname(node_modules), pname, version):
-        os.remove(dep)
-        continue
-
-    print(f"copying {dep}")
-    os.rename(dep, f"{dep}.bac")
-    os.mkdir(dep)
-    contents = os.listdir(f"{dep}.bac")
-    if contents != []:
-      for node in contents:
-        if os.path.isdir(f"{dep}.bac/{node}"):
-          shutil.copytree(f"{dep}.bac/{node}", f"{dep}/{node}", symlinks=True)
-          if os.path.isdir(f"{dep}/node_modules"):
-            symlinks_to_copies(f"{dep}/node_modules")
-        else:
-          shutil.copy(f"{dep}.bac/{node}", f"{dep}/{node}")
-    os.remove(f"{dep}.bac")
-    symlink_bin(f"{bin_dir}", dep, package_json)
-
-
-def symlink_direct_bins():
-  deps = []
-  package_json_file = get_package_json(f"{os.path.abspath('.')}")
-
-  if package_json_file:
-    if 'devDependencies' in package_json_file and package_json_file['devDependencies']:
-      for dep,_ in package_json_file['devDependencies'].items():
-        deps.append(dep)
-    if 'dependencies' in package_json_file and package_json_file['dependencies']:
-      for dep,_ in package_json_file['dependencies'].items():
-        deps.append(dep)
-
-  for name in deps:
-    package_location = f"{root}/{name}"
-    package_json = get_package_json(package_location)
-    symlink_bin(f"{bin_dir}", package_location, package_json, force=True)
-
-
-# install direct deps
-install_direct_dependencies()
-
-# symlink non-colliding deps
-symlink_sub_dependencies()
-
-# symlinks to copies
-if os.environ.get('installMethod') == 'copy':
-  symlinks_to_copies(root)
-
-# symlink direct deps bins
-symlink_direct_bins()
diff --git a/src/subsystems/nodejs/builders/granular/link-bins.py b/src/subsystems/nodejs/builders/granular/link-bins.py
index 26c28629d9..7008871163 100644
--- a/src/subsystems/nodejs/builders/granular/link-bins.py
+++ b/src/subsystems/nodejs/builders/granular/link-bins.py
@@ -35,7 +35,5 @@ def link(name, relpath):
       for name, relpath in bin.items():
         link(name, relpath)
 
-# symlink current packages executables to $nodeModules/.bin
-symlink_bin(f'{out}/lib/node_modules/.bin/', package_json)
 # symlink current packages executables to $out/bin
 symlink_bin(f'{out}/bin/', package_json)
diff --git a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js b/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js
deleted file mode 100644
index 1ccd3db042..0000000000
--- a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js
+++ /dev/null
@@ -1,19 +0,0 @@
-try {
-  console.log(require.resolve("typescript"));
-} catch(e) {
-  console.error("typescript is not found");
-  process.exit(e.code);
-}
-
-const ts = require("typescript")
-const fs = require('fs')
-
-try {
-  const data = fs.readFileSync('tsconfig.json', 'utf8')
-} catch (err) {
-  console.error(err)
-}
-
-config = ts.parseConfigFileTextToJson(data)
-newdata = JSON.stringify(config)
-fs.writeFileSync('tsconfig.json', newdata);