Skip to content

Commit

Permalink
wip working
Browse files Browse the repository at this point in the history
  • Loading branch information
@wmertens
wmertens committed Aug 23, 2022
1 parent c2e4006 commit 37571f6
Show file tree
Hide file tree
Showing 8 changed files with 240 additions and 216 deletions.
1 change: 1 addition & 0 deletions flake.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@

# a dev shell for working on dream2nix
# use via 'nix develop . -c $SHELL'
# TODO only for the current system?
devShells = forAllSystems (system: pkgs: let
makeDevshell = import "${inp.devshell}/modules" pkgs;
mkShell = config:
Expand Down
212 changes: 109 additions & 103 deletions src/subsystems/nodejs/builders/granular/default.nix
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,3 @@
# TODO if build script in main, run with devModules
# TODO split build in multiple derivations
# - first derivation:
# - unpack without modules
# - if main, optional build step with devModules (discover with runcmd)
# - optional npm install w/ all deps + source-only cyclic
# - only inject deps when needed to prevent rebuilds. Remove deps after build.
# - second derivation: link with modules. Cyclic deps are joined.
{...}: {
type = "pure";

Expand Down Expand Up @@ -53,7 +45,8 @@
b = builtins;
l = lib // builtins;

nodejsVersion = subsystemAttrs.nodejsVersion;
nodejsVersion = subsystemAttrs.nodejsVersion or null;
transitiveBinaries = subsystemAttrs.transitiveBinaries or null;

isMainPackage = name: version:
(args.packages."${name}" or null) == version;
Expand All @@ -67,11 +60,6 @@
or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs")
else pkgs.nodejs;

nodeSources = runCommandLocal "node-sources" {} ''
tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
mv node-* $out
'';

allPackages =
lib.mapAttrs
(name: versions:
Expand Down Expand Up @@ -136,14 +124,8 @@
cp -r --no-preserve=mode $electronDist $TMP/electron
# configure electron toolchain
${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
${pkgs.jq}/bin/jq ".build.linux.target = \"dir\"" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
${pkgs.jq}/bin/jq ".build.npmRebuild = false" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
jq ".build.electronDist = \"$TMP/electron\" | .build.linux.target = \"dir\" | .build.npmRebuild = false" package.json > package.json.tmp
mv package.json.tmp package.json
# execute electron-rebuild if available
export headers=http://localhost:45034/
Expand Down Expand Up @@ -207,7 +189,7 @@
(
dep: let
p = allPackages."${dep.name}"."${dep.version}";
s = p.sourceInfo;
s = p.extraInfo;
in
# this dep is a cyclee
!(isCyclee dep.name dep.version)
Expand Down Expand Up @@ -242,7 +224,7 @@
myDeps =
lib.filter
(dep: let
s = dep.sourceInfo;
s = dep.extraInfo;
in
(withOptionals || !(s.optional or false))
&& (!isMain || (withDev || !(s.dev or false))))
Expand Down Expand Up @@ -311,7 +293,6 @@
# if noDev was used, these are just the prod modules
devModules = makeModules {withDev = true;};

# TODO why is this needed? Seems to work without
passthruDeps =
l.listToAttrs
(l.forEach deps
Expand Down Expand Up @@ -347,35 +328,35 @@
then null
else pkgs."electron_${electronVersionMajor}".headers;

sourceInfo = let
e = getSourceSpec name version;
try = builtins.tryEval (builtins.deepSeq e e);
in
if try.success
then try.value
else {};
hasInstall = !(sourceInfo.noInstall or false);
hasExtraInfo = subsystemAttrs ? extraInfo;
extraInfo = subsystemAttrs.extraInfo.${name}.${version} or {};
# If the translator doesn't provide extraInfo, assume scripts
hasInstall =
if hasExtraInfo
then extraInfo.hasInstallScript or false
else true;
isMain = isMainPackage name version;

pkg = produceDerivation name (stdenv.mkDerivation rec {
inherit
dependenciesJson
electronHeaders
nodeSources
version
transitiveBinaries
prodModules
;

packageName = name;

inherit pname;

# TODO why is this needed? It works without it?
# passthru.dependencies = passthruDeps;
passthru.dependencies = passthruDeps;

passthru.devShell = import ./devShell.nix {
inherit mkShell nodejs devModules;
};

passthru.extraInfo = extraInfo;
/*
For top-level packages install dependencies as full copies, as this
reduces errors with build tooling that doesn't cope well with
Expand All @@ -390,27 +371,45 @@
electronAppDir = ".";

# only run build on the main package
runBuild = isMain;
runBuild = isMain && (subsystemAttrs.hasBuildScript or true);

src = getSource name version;
# can be overridden to define alternative install command
# (defaults to npm install steps)
buildScript = null;
shouldBuild = hasInstall || runBuild || buildScript != null || electronHeaders != null;
buildModules =
if runBuild
then devModules
else prodModules;
nodeSources =
if shouldBuild
then nodejs
else null;

nativeBuildInputs = [makeWrapper];
# We don't need unpacked sources
src = let t = getSource name version; in t.original or t;

buildInputs = [jq nodejs python3];
nativeBuildInputs =
if shouldBuild
then [makeWrapper]
else [];

# We must provide nodejs even when not building to allow
# patchShebangs to find it for binaries
buildInputs =
if shouldBuild || (!hasExtraInfo || (extraInfo ? bin))
then [jq nodejs python3]
else [python3];

# prevents running into ulimits
passAsFile = ["dependenciesJson"];

preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];

# can be overridden to define alternative install command
# (defaults to npm install steps)
buildScript = null;
shouldBuild = hasInstall || buildScript != null || electronHeaders != null;

# python script to modify some metadata to support installation
# (see comments below on d2nPatchPhase)
fixPackage = "${./fix-package.py}";
linkBins = "${./link-bins.py}";

# costs performance and doesn't seem beneficial in most scenarios
dontStrip = true;
Expand All @@ -434,13 +433,10 @@
}
'';

# TODO: upstream fix to nixpkgs
# https://github.com/NixOS/nixpkgs/pull/50961#issuecomment-449638192
# example which requires this:
# https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz
unpackCmd =
if lib.hasSuffix ".tgz" src
then "tar --delay-directory-restore -xf $src"
else null;
TAR_OPTIONS = "--delay-directory-restore";

unpackPhase = ''
runHook preUnpack
Expand Down Expand Up @@ -468,19 +464,17 @@
# Figure out what directory has been unpacked
export packageDir="$(find . -maxdepth 1 -type d | tail -1)"
# TODO why is this needed?
# find "$packageDir" -type d -exec chmod u+x {} \;
# Restore write permissions
chmod -R u+w -- "$packageDir"
# Ensure write + directory execute permissions
chmod -R u+w,a+X -- "$packageDir"
# Move the extracted tarball into the output folder
mv -- "$packageDir" "$sourceRoot"
elif [ -d "$src" ]
then
export strippedName="$(stripHash $src)"
# Restore write permissions
chmod -R u+w -- "$strippedName"
# Ensure write + directory execute permissions
chmod -R u+w,a+X -- "$strippedName"
# Move the extracted directory into the output folder
mv -- "$strippedName" "$sourceRoot"
Expand All @@ -507,10 +501,6 @@
# - If package-lock.json exists, it is deleted, as it might conflict
# with the parent package-lock.json.
d2nPatchPhase = ''
if [ -z "$shouldBuild" ] && [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
shouldBuild=1
fi
# delete package-lock.json as it can lead to conflicts
rm -f package-lock.json
Expand All @@ -533,23 +523,16 @@
configurePhase = ''
runHook preConfigure
${
if prodModules != null
then ''
if [ -L $sourceRoot/node_modules ] || [ -e $sourceRoot/node_modules ]; then
echo Warning: The source $sourceRoot includes a node_modules directory. Replacing. >&2
rm -rf $sourceRoot/node_modules
fi
ln -s ${prodModules} $sourceRoot/node_modules
if [ -d ${prodModules}/.bin ]; then
export PATH="$PATH:$sourceRoot/node_modules/.bin"
# pass down transitive binaries, like npm does
# all links are absolute so we can just copy
cp -af --no-preserve=mode ${prodModules}/.bin/. $out/bin/.
fi
''
else ""
}
if [ -n "$buildModules" ]; then
if [ -L $sourceRoot/node_modules ] || [ -e $sourceRoot/node_modules ]; then
echo Warning: The source $sourceRoot includes a node_modules directory. Replacing. >&2
rm -rf $sourceRoot/node_modules
fi
ln -s $buildModules $sourceRoot/node_modules
if [ -d $buildModules/.bin ]; then
export PATH="$PATH:$sourceRoot/node_modules/.bin"
fi
fi
${
# Here we copy cyclee deps into the cyclehead node_modules
# so the cyclic deps can find each other
Expand Down Expand Up @@ -580,43 +563,67 @@
# Runs the install command which defaults to 'npm run postinstall'.
# Allows using custom install command by overriding 'buildScript'.
# TODO this logic supposes a build script, which is not documented
buildPhase = ''
runHook preBuild
if [ -n "$shouldBuild" ]; then
# execute electron-rebuild
if [ -n "$electronHeaders" ]; then
echo "executing electron-rebuild"
${electron-rebuild}
fi
# for installing, we only need to run `npm run install` (pre and post scripts run automatically)
# https://github.com/npm/npm/issues/5919
# TODO build first if has build, give it devModules during build

buildPhase =
if shouldBuild
then ''
set -x
runHook preBuild
if [ -n "$shouldBuild" ]; then
# execute electron-rebuild
if [ -n "$electronHeaders" ]; then
echo "executing electron-rebuild"
${electron-rebuild}
fi
# execute install command
if [ -n "$buildScript" ]; then
if [ -f "$buildScript" ]; then
$buildScript
# execute install command
if [ -n "$buildScript" ]; then
if [ -f "$buildScript" ]; then
$buildScript
else
eval "$buildScript"
fi
else
eval "$buildScript"
if [ -n "$runBuild" ]; then
# by default, only for top level packages, `npm run build` is executed
npm run --if-present build
fi
# This seems to be the only script that needs running on install
npm --omit=dev --offline --nodedir=$nodeSources run --if-present install
fi
elif [ -n "$runBuild" ]; then
# by default, only for top level packages, `npm run build` is executed
npm run --if-present build
else
npm --omit=dev --offline --nodedir=$nodeSources run --if-present preinstall
npm --omit=dev --offline --nodedir=$nodeSources run --if-present install
npm --omit=dev --offline --nodedir=$nodeSources run --if-present postinstall
fi
fi
runHook postBuild
'';
runHook postBuild
set +x
''
else "true";

# Symlinks executables and manual pages to correct directories
installPhase = ''
runHook preInstall
if [ "$buildModules" != "$prodModules" ]; then
if [ -n "$prodModules" ]; then
ln -sf $prodModules $sourceRoot/node_modules
else
rm $sourceRoot/node_modules
fi
fi
echo "Symlinking bin entries from package.json"
python $linkBins
if [ -n "$transitiveBinaries" ]; then
# pass down transitive binaries, like npm does
# all links are absolute so we can just copy
cp -af --no-preserve=mode $prodModules/.bin/. $out/bin/.
fi
if rmdir $out/bin 2>/dev/null; then
# we didn't install any binaries
rm $nodeModules/.bin
Expand Down Expand Up @@ -650,8 +657,7 @@
'';
});
in
pkg
// {inherit sourceInfo;};
pkg;
in
outputs;
}
Loading

0 comments on commit 37571f6

Please sign in to comment.