From 4de6aaea5dd3689ba27258798b7fa92d88eea257 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Feb 2018 18:26:47 -0500 Subject: [PATCH 1/3] Allow `*.nix` just within subdirectories --- default.nix | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index d88c89e..353b97e 100644 --- a/default.nix +++ b/default.nix @@ -1,7 +1,20 @@ { pkgs ? import {}, revCount, shortRev }: let lib = pkgs.lib; - sources = lib.sourceFilesBySuffices ./. [ ".xml" ".txt" ]; + + sources = let + + # We want nix examples, but not the top level nix to build things + noTopLevelNix = path: type: let + relPath = lib.removePrefix (toString ./. + "/") (toString path); + in builtins.match "[^/]*\.nix" relPath == null; + + extensions = [ ".xml" ".txt" ".nix" ".bash" ]; + + in lib.cleanSourceWith { + filter = noTopLevelNix; + src = lib.sourceFilesBySuffices ./. extensions; + }; combined = pkgs.runCommand "nix-pills-combined" { From aca2170ce0057043d5b9797fd0cbada1006e96ea Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Feb 2018 17:00:43 -0500 Subject: [PATCH 2/3] Pill 19: Clean up working a --- pills/19-fundamentals-of-stdenv.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pills/19-fundamentals-of-stdenv.xml b/pills/19-fundamentals-of-stdenv.xml index 2a4b711..423f541 100644 --- a/pills/19-fundamentals-of-stdenv.xml +++ b/pills/19-fundamentals-of-stdenv.xml @@ -15,7 +15,7 @@ - The stdenv is not a special derivation, but it's very important for the nixpkgs repository. It serves as base for packaging software. It is used to pull in dependencies such as the GCC toolchain, GNU make, core utilities, patch and diff utilities, and so on. Basic tools needed to compile a huge pile of software currently present in nixpkgs. + The stdenv is not a special derivation to Nix, but it's very important for the nixpkgs repository. It serves as base for packaging software. It is used to pull in dependencies such as the GCC toolchain, GNU make, core utilities, patch and diff utilities, and so on. Basic tools needed to compile a huge pile of software currently present in nixpkgs.
What is stdenv @@ -26,7 +26,7 @@ - It has just two files: /setup and /nix-support/propagated-user-env-packages. Don't care about the latter, it's even empty. The important file is /setup. + It has just two files: /setup and /nix-support/propagated-user-env-packages. Don't care about the latter; it's empty, in fact. The important file is /setup. @@ -52,7 +52,7 @@ - The hardcoded toolchain and utilities are used to initially fill up the environment variables so that it's more pleasant to run common commands, similarly but not equal like we did with our builder with baseInputs and buildInputs. + The hardcoded toolchain and utilities are used to initially fill up the environment variables so that it's more pleasant to run common commands, similar to what we did with our builder with baseInputs and buildInputs. @@ -64,7 +64,8 @@ - Every phase has hooks to run commands before and after the phase has been executed. Phases can be overwritten, reordered, whatever, it's just bash code. + Every phase has hooks to run commands before and after the phase has been executed. + Phases can be overwritten, reordered, whatever, it's just bash code. @@ -114,7 +115,7 @@ The stdenv.mkDerivation builder - Let's take a look at the builder used by mkDerivation. You can read the code here in nixpkgs: + Let's take a look at the builder used by mkDerivation. You can read the code here in nixpkgs: From 4c513daccd1892b4606a49e7c12a96be3ae21aec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Feb 2018 17:27:25 -0500 Subject: [PATCH 3/3] Pill 20: Init according 19's teaser --- book.xml | 1 + pills/19-fundamentals-of-stdenv.xml | 3 +- pills/20-basic-dependencies-and-hooks.xml | 179 ++++++++++++++++++++++ pills/20/build-inputs-0.bash | 4 + pills/20/build-inputs-1.bash | 14 ++ pills/20/build-inputs-2.bash | 3 + pills/20/build-inputs-3.bash | 9 ++ pills/20/env-hooks-0.bash | 10 ++ pills/20/env-hooks-1.bash | 7 + pills/20/propagated-build-inputs-0.bash | 12 ++ pills/20/propagated-build-inputs-1.bash | 11 ++ pills/20/propagated-build-inputs-3.bash | 4 + pills/20/setup-hooks-0.bash | 12 ++ pills/20/three-hellos.nix | 40 +++++ pills/20/two-hellos.nix | 30 ++++ 15 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 pills/20-basic-dependencies-and-hooks.xml create mode 100644 pills/20/build-inputs-0.bash create mode 100644 pills/20/build-inputs-1.bash create mode 100644 pills/20/build-inputs-2.bash create mode 100644 pills/20/build-inputs-3.bash create mode 100644 pills/20/env-hooks-0.bash create mode 100644 pills/20/env-hooks-1.bash create mode 100644 pills/20/propagated-build-inputs-0.bash create mode 100644 pills/20/propagated-build-inputs-1.bash create mode 100644 pills/20/propagated-build-inputs-3.bash create mode 100644 pills/20/setup-hooks-0.bash create mode 100644 pills/20/three-hellos.nix create mode 100644 pills/20/two-hellos.nix diff --git a/book.xml b/book.xml index ca08c59..d961fea 100644 --- a/book.xml +++ b/book.xml @@ -57,4 +57,5 @@ + diff --git a/pills/19-fundamentals-of-stdenv.xml b/pills/19-fundamentals-of-stdenv.xml index 423f541..40562a6 100644 --- a/pills/19-fundamentals-of-stdenv.xml +++ b/pills/19-fundamentals-of-stdenv.xml @@ -179,7 +179,8 @@ Next pill... - ...we will talk about how to add dependencies to our packages, buildInputs, propagatedBuildInputs and setup hooks. These three concepts are at the base of the current nixpkgs packages composition. + ...we will talk about how to add dependencies to our packages with buildInputs and propagatedBuildInputs, and influence downstream builds with setup hooks and env hooks. + These concepts are crucial to how nixpkgs packages are composed.
diff --git a/pills/20-basic-dependencies-and-hooks.xml b/pills/20-basic-dependencies-and-hooks.xml new file mode 100644 index 0000000..0b80931 --- /dev/null +++ b/pills/20-basic-dependencies-and-hooks.xml @@ -0,0 +1,179 @@ + + + Basic Dependencies and Hooks + + + Welcome to the 20th Nix pill. + In the previous 19th pill we introduced Nixpkgs' Stdenv, including setup.sh script, default-builder.sh helper script, and stdenv.mkDerivation builder. + We focused on how stdenv is put together, and how it's used, an a bit about the phases of genericBuild. + + + + This time, we'll focus on the interaction of packages built with stdenv.mkDerivation. + Packages need to depend on each other, of course. + For this we have buildInputs and propagatedBuildInputs attributes. + We've also found that dependencies sometimes need to influence their dependents in ways the dependents can't or shouldn't predict. + For this we have setup hooks and env hooks. + Together, these 4 concepts support almost all build-time package interactions. + + + + This Nix pill is going to teach Nixpkgs a bit differently. + The Nix pills before this one have used the most recent version of Nixpkgs when discussing it. + But the dependencies and hooks infrastructure has for a few years been increasingly complicated to support cross compilation, with a final boost of complexity in recently in 2017. + While the added mechanism shouldn't be too confusing after the core concepts are taught, they just get in the way before, so this Nix Pill will go all the way back to 2009 with commit 6675f0a5. + This is the last version of Stdenv before extra dependency and hook types for cross compilation were created. + + +
+ The <varname>buildInputs</varname> Attribute + + + For the simplest dependencies where the current package directly needs another, we use the buildInputs attribute. + This is exactly the pattern in taught with our builder in Pill 8. + To demo this, lets build GNU Hello, and then another package which provides a shell script that execs it. + + + + + This works because stdenv contains something like: + + where findInputs is defined like: + + then after this is run: + + where addToEnv is defined like: + + + The addToSearchPath call adds $1/bin to _PATH if the former exists (code here). + + With the real hello on the path, the installPhase should hopefully make sense. + +
+ +
+ The <varname>propagatedBuildInputs</varname> Attribute + + + The buildInputs covers direct dependencies, but what about indirect dependencies where one package needs a second package which needs a third? + Nix itself handles this just fine, understanding various dependency closures as covered in previous builds. + But what about the conveniences that buildInputs provides, namely accumulating in pkgs environment variable and inclusion of pkg/bin directories on the PATH? + For this, Stdenv provides the propagatedBuildInputs: + + See how the intermediate package has a propagatedBuildInputs dependency, but the wrapper only needs a buildInputs dependency on the intermediary. + + + + How does this work? + You might think we do something in Nix, but actually its done not at eval time but at build time in bash. + lets look at part of the fixupPhase of Stdenv: + + + + This dumps the propagated build inputs in a so-named file in $out/nix-support/. + Then, back in findInputs look at the lines at the bottom we elided before: + + + + See how findInputs is actually recursive, looking at the propagated build inputs of each dependency, and those dependencies' propagated build inputs, etc. + + + + We actually simplified the findInputs call site from before; propagatedBuildInputs is also looped over in reality: + + This demonstrates an important point. For the current package alone, it doesn't matter whether a dependency is propagated or not. + It will be processed the same way: called with findInputs and addToEnv. + (The packages discovered by findInputs, which are also accumulated in pkgs and passed to addToEnv, are also the same in both cases.) + Downstream however, it certainly does matter because only the propagated immediate dependencies are put in the $out/nix-support/propagated-build-inputs. + +
+ +
+ Setup Hooks + + + As we mentioned above, sometimes dependencies need to influence the packages that use them in ways other than just being a dependency. + + + + We can now be precise and consider what addToEnv does alone the minimal treatment of a dependency: + i.e. a package that is just a dependency would only have addToEnv applied to it. + + + + propagatedBuildInputs can actually be seen as an example of this: + packages using that are effectively "injecting" those dependencies as extra buildInputs in their downstream dependents. + + But in general, a dependency might to affect the packages it depends on in arbitrary ways. + arbitrary is the key word here + We could teach setup.sh things about upstream packages like pkg/nix-support/propagated-build-inputs, but not arbitrary interactions. + + + + Setup hooks are the basic building block we have for this. + In nixpkgs, a "hook" is basically a bash callback, and a setup hook is no exception. + Let's look at the last part of findInputs we haven't covered: + + If a package includes the path pkg/nix-support/setup-hook, it will be sourced by any Stdenv-based build including that as a dependency. + + + + This is strictly more general than any of the other mechanisms introduced in this chapter. + For example, try writing a setup hook that has the same effect as a propagatedBuildInputs entry. + One can almost think of this as an escape hatch around Nix's normal isolation guarantees, and the principle that dependencies are immutable and inert. + We're not actually doing something unsafe or modifying dependencies, but we are allowing arbitrary ad-hoc behavior. + For this reason, setup-hooks should only be used as a last resort. + +
+ +
+ Environment Hooks + + + As a final convenience, we have environment hooks. + Recall in Pill 12 how we created NIX_CFLAGS_COMPILE for -I flags and NIX_LDFLAGS for -L flags, in a similar manner to how we prepared the PATH. + One point of ugliness was how anti-modular this was. + It makes sense to build the PATH in generic builder, because the PATH is used by the shell, and the generic builder is intrinsically tied to the shell. + But -I and -L flags are only relevant to the C compiler. + The stdenv isn't wedded to including a C compiler (though it does by default), and there are other compilers too which may take completely different flags. + + + + As a first step, we can move that logic to a setup hook on the C compiler; + indeed that's just what we do in CC Wrapper. + + + . It was called GCC Wrapper in the version of nixpkgs we're using in this pill; Darwin and Clang support hadn't yet motivated the rename. + + + But this pattern comes up fairly often, so somebody decided to add some helper support to reduce boilerplate. + + + + The other half of addToEnv is: + + See how for every package we iterate through the contents of envHooks, and apply every function within to the package in question. + The idea is one can write a setup hook like: + + and if one dependency has that setup hook then all of them will be so echoed. + Allowing dependencies to learn about their sibling dependencies is exactly what compiler need. + + +
+ +
+ Next pill... + + ...I'm not sure! + We could talk the additional dependency types and hooks which cross compilation necessitates, building on our knowledge here to cover stdenv as it works today. + We could talk about how nixpkgs is bootstrapped. + Or we could talk about how localSystem and crossSystem are elaborated into the buildPlatform, hostPlatform, and targetPlatform each bootstrapping stage receives. + Let us know which most interests you! + +
+ + diff --git a/pills/20/build-inputs-0.bash b/pills/20/build-inputs-0.bash new file mode 100644 index 0000000..1188faf --- /dev/null +++ b/pills/20/build-inputs-0.bash @@ -0,0 +1,4 @@ +pkgs="" +for i in $buildInputs; do + findInputs $i +done diff --git a/pills/20/build-inputs-1.bash b/pills/20/build-inputs-1.bash new file mode 100644 index 0000000..1b09b42 --- /dev/null +++ b/pills/20/build-inputs-1.bash @@ -0,0 +1,14 @@ +findInputs() { + local pkg=$1 + + ## Don't need to repeat already processed package + case $pkgs in + *\ $pkg\ *) + return 0 + ;; + esac + + pkgs="$pkgs $pkg " + + ## More goes here in reality that we can ignore for now. +} diff --git a/pills/20/build-inputs-2.bash b/pills/20/build-inputs-2.bash new file mode 100644 index 0000000..d3db58c --- /dev/null +++ b/pills/20/build-inputs-2.bash @@ -0,0 +1,3 @@ +for i in $pkgs; do + addToEnv $i +done diff --git a/pills/20/build-inputs-3.bash b/pills/20/build-inputs-3.bash new file mode 100644 index 0000000..80e3207 --- /dev/null +++ b/pills/20/build-inputs-3.bash @@ -0,0 +1,9 @@ +addToEnv() { + local pkg=$1 + + if test -d $1/bin; then + addToSearchPath _PATH $1/bin + fi + + ## More goes here in reality that we can ignore for now. +} diff --git a/pills/20/env-hooks-0.bash b/pills/20/env-hooks-0.bash new file mode 100644 index 0000000..68c0de6 --- /dev/null +++ b/pills/20/env-hooks-0.bash @@ -0,0 +1,10 @@ +addToEnv() { + local pkg=$1 + + ## More goes here in reality that we can ignore for now. + + # Run the package-specific hooks set by the setup-hook scripts. + for i in "${envHooks[@]}"; do + $i $pkg + done +} diff --git a/pills/20/env-hooks-1.bash b/pills/20/env-hooks-1.bash new file mode 100644 index 0000000..f64daf9 --- /dev/null +++ b/pills/20/env-hooks-1.bash @@ -0,0 +1,7 @@ +anEnvHook() { + local pkg=$1 + + echo "I'm depending on \"$pkg\"" +} + +envHooks+=(anEnvHook) diff --git a/pills/20/propagated-build-inputs-0.bash b/pills/20/propagated-build-inputs-0.bash new file mode 100644 index 0000000..ee3a526 --- /dev/null +++ b/pills/20/propagated-build-inputs-0.bash @@ -0,0 +1,12 @@ +fixupPhase() { + + ## Elided + + if test -n "$propagatedBuildInputs"; then + ensureDir "$out/nix-support" + echo "$propagatedBuildInputs" > "$out/nix-support/propagated-build-inputs" + fi + + ## Elided + +} diff --git a/pills/20/propagated-build-inputs-1.bash b/pills/20/propagated-build-inputs-1.bash new file mode 100644 index 0000000..5c4b217 --- /dev/null +++ b/pills/20/propagated-build-inputs-1.bash @@ -0,0 +1,11 @@ +findInputs() { + local pkg=$1 + + ## More goes here in reality that we can ignore for now. + + if test -f $pkg/nix-support/propagated-build-inputs; then + for i in $(cat $pkg/nix-support/propagated-build-inputs); do + findInputs $i + done + fi +} diff --git a/pills/20/propagated-build-inputs-3.bash b/pills/20/propagated-build-inputs-3.bash new file mode 100644 index 0000000..a5e5e76 --- /dev/null +++ b/pills/20/propagated-build-inputs-3.bash @@ -0,0 +1,4 @@ +pkgs="" +for i in $buildInputs $propagatedBuildInputs; do + findInputs $i +done diff --git a/pills/20/setup-hooks-0.bash b/pills/20/setup-hooks-0.bash new file mode 100644 index 0000000..a0b5de9 --- /dev/null +++ b/pills/20/setup-hooks-0.bash @@ -0,0 +1,12 @@ +findInputs() { + local pkg=$1 + + ## More goes here in reality that we can ignore for now. + + if test -f $pkg/nix-support/setup-hook; then + source $pkg/nix-support/setup-hook + fi + + ## More goes here in reality that we can ignore for now. + +} diff --git a/pills/20/three-hellos.nix b/pills/20/three-hellos.nix new file mode 100644 index 0000000..246c130 --- /dev/null +++ b/pills/20/three-hellos.nix @@ -0,0 +1,40 @@ +let + + nixpkgs = import (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/6675f0a52c0962042a1000c7f20e887d0d26ae25.tar.gz"; + }) {}; + + inherit (nixpkgs) stdenv fetchurl; + + actualHello = stdenv.mkDerivation { + name = "hello-2.3"; + + src = fetchurl { + url = mirror://gnu/hello/hello-2.3.tar.bz2; + sha256 = "0c7vijq8y68bpr7g6dh1gny0bff8qq81vnp4ch8pjzvg56wb3js1"; + }; + }; + + intermediary = stdenv.mkDerivation { + name = "middle-man"; + + propagatedBuildInputs = [ actualHello ]; + + installPhase = '' + mkdir -p "$out" + ''; + }; + + wrappedHello = stdenv.mkDerivation { + name = "hello-wrapper"; + + buildInputs = [ intermediary ]; + + installPhase = '' + mkdir -p "$out/bin" + echo "#! ${stdenv.shell}" >> "$out/bin/hello" + echo "exec $(which hello)" >> "$out/bin/hello" + ''; + }; + +in wrappedHello diff --git a/pills/20/two-hellos.nix b/pills/20/two-hellos.nix new file mode 100644 index 0000000..6e5d309 --- /dev/null +++ b/pills/20/two-hellos.nix @@ -0,0 +1,30 @@ +let + + nixpkgs = import (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/6675f0a52c0962042a1000c7f20e887d0d26ae25.tar.gz"; + }) {}; + + inherit (nixpkgs) stdenv fetchurl; + + actualHello = stdenv.mkDerivation { + name = "hello-2.3"; + + src = fetchurl { + url = mirror://gnu/hello/hello-2.3.tar.bz2; + sha256 = "0c7vijq8y68bpr7g6dh1gny0bff8qq81vnp4ch8pjzvg56wb3js1"; + }; + }; + + wrappedHello = stdenv.mkDerivation { + name = "hello-wrapper"; + + buildInputs = [ actualHello ]; + + installPhase = '' + mkdir -p "$out/bin" + echo "#! ${stdenv.shell}" >> "$out/bin/hello" + echo "exec $(which hello)" >> "$out/bin/hello" + ''; + }; + +in wrappedHello