Merge 34f2477d2e into 806a91f7bf

Clean up the `package.nix` files

doc/manual/src/release-notes/rl-next.md

@@ -0,0 +1,4 @@
+# Release X.Y (202?-??-??)
+
+- Add a `load-limit` setting to control builder parallelism. This has
+  also been backported to the 2.18 and later release branches.

packaging/dependencies.nix

@@ -38,6 +38,10 @@ let
   # Indirection for Nixpkgs to override when package.nix files are vendored
   filesetToSource = lib.fileset.toSource;
 
+  /** Given a set of layers, create a mkDerivation-like function */
+  mkPackageBuilder = exts: userFn:
+    stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn);
+
   localSourceLayer = finalAttrs: prevAttrs:
     let
       workDirPath =
@@ -60,6 +64,28 @@ let
       workDir = null;
     };
 
+  mesonLayer = finalAttrs: prevAttrs:
+    {
+      nativeBuildInputs = [
+        pkgs.buildPackages.meson
+        pkgs.buildPackages.ninja
+      ] ++ prevAttrs.nativeBuildInputs or [];
+    };
+
+  mesonBuildLayer = finalAttrs: prevAttrs:
+    {
+      nativeBuildInputs = prevAttrs.nativeBuildInputs or [] ++ [
+        pkgs.buildPackages.pkg-config
+      ];
+      separateDebugInfo = !stdenv.hostPlatform.isStatic;
+      hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+    };
+
+  mesonLibraryLayer = finalAttrs: prevAttrs:
+    {
+      outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
+    };
+
   # Work around weird `--as-needed` linker behavior with BSD, see
   # https://github.com/mesonbuild/meson/issues/3593
   bsdNoLinkAsNeeded = finalAttrs: prevAttrs:
@@ -172,14 +198,27 @@ scope: {
 
   inherit resolvePath filesetToSource;
 
-  mkMesonDerivation = f: let
-    exts = [
+  mkMesonDerivation =
+    mkPackageBuilder [
+      miscGoodPractice
+      localSourceLayer
+      mesonLayer
+    ];
+  mkMesonExecutable =
+    mkPackageBuilder [
       miscGoodPractice
       bsdNoLinkAsNeeded
       localSourceLayer
+      mesonLayer
+      mesonBuildLayer
+    ];
+  mkMesonLibrary =
+    mkPackageBuilder [
+      miscGoodPractice
+      bsdNoLinkAsNeeded
+      localSourceLayer
+      mesonLayer
+      mesonBuildLayer
+      mesonLibraryLayer
     ];
-  in stdenv.mkDerivation
-   (lib.extends
-     (lib.foldr lib.composeExtensions (_: _: {}) exts)
-     f);
 }

src/external-api-docs/package.nix

@@ -1,8 +1,6 @@
 { lib
 , mkMesonDerivation
 
-, meson
-, ninja
 , doxygen
 
 # Configuration Options
@@ -37,8 +35,6 @@ mkMesonDerivation (finalAttrs: {
     ];
 
   nativeBuildInputs = [
-    meson
-    ninja
     doxygen
   ];
 

src/internal-api-docs/package.nix

@@ -1,8 +1,6 @@
 { lib
 , mkMesonDerivation
 
-, meson
-, ninja
 , doxygen
 
 # Configuration Options
@@ -32,8 +30,6 @@ mkMesonDerivation (finalAttrs: {
   ];
 
   nativeBuildInputs = [
-    meson
-    ninja
     doxygen
   ];
 

src/libcmd/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 , nix-store
@@ -38,7 +33,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-cmd";
   inherit version;
 
@@ -54,14 +49,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     ({ inherit editline readline; }.${readlineFlavor})
   ] ++ lib.optional enableMarkdown lowdown;
@@ -93,10 +80,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libexpr-c/package.nix

@@ -1,10 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-store-c
 , nix-expr
@@ -18,7 +14,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-expr-c";
   inherit version;
 
@@ -35,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-store-c
     nix-expr
@@ -63,10 +51,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libexpr/package.nix

@@ -1,11 +1,7 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
+, mkMesonLibrary
 
-, meson
-, ninja
-, pkg-config
 , bison
 , flex
 , cmake # for resolving toml11 dep
@@ -38,7 +34,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-expr";
   inherit version;
 
@@ -55,15 +51,13 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
     ./lexer.l
     ./parser.y
-    (fileset.fileFilter (file: file.hasExt "nix") ./.)
+    (fileset.difference
+      (fileset.fileFilter (file: file.hasExt "nix") ./.)
+      ./package.nix
+    )
   ];
 
-  outputs = [ "out" "dev" ];
-
   nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
     bison
     flex
     cmake
@@ -102,10 +96,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libfetchers/git-utils.cc

@@ -208,7 +208,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
 static void initRepoAtomically(std::filesystem::path &path, bool bare) {
     if (pathExists(path.string())) return;
 
-    Path tmpDir = createTempDir(std::filesystem::path(path).parent_path());
+    Path tmpDir = createTempDir(os_string_to_string(PathViewNG { std::filesystem::path(path).parent_path() }));
     AutoDelete delTmpDir(tmpDir, true);
     Repository tmpRepo;
 
@@ -977,8 +977,24 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
     void pushBuilder(std::string name)
     {
+        const git_tree_entry * entry;
+        Tree prevTree = nullptr;
+
+        if (!pendingDirs.empty() &&
+            (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str())))
+        {
+            /* Clone a tree that we've already finished. This happens
+               if a tarball has directory entries that are not
+               contiguous. */
+            if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+                throw Error("parent of '%s' is not a directory", name);
+
+            if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(prevTree), *repo, entry))
+                throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
+        }
+
         git_treebuilder * b;
-        if (git_treebuilder_new(&b, *repo, nullptr))
+        if (git_treebuilder_new(&b, *repo, prevTree.get()))
             throw Error("creating a tree builder: %s", git_error_last()->message);
         pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) });
     };

src/libfetchers/package.nix

@@ -1,17 +1,11 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 , nix-store
 , nlohmann_json
 , libgit2
-, man
 
 # Configuration Options
 
@@ -22,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-fetchers";
   inherit version;
 
@@ -37,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     libgit2
   ];
@@ -67,10 +53,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libfetchers/tarball.cc

@@ -90,6 +90,7 @@ DownloadFileResult downloadFile(
     /* Cache metadata for all URLs in the redirect chain. */
     for (auto & url : res.urls) {
         key.second.insert_or_assign("url", url);
+        assert(!res.urls.empty());
         infoAttrs.insert_or_assign("url", *res.urls.rbegin());
         getCache()->upsert(key, *store, infoAttrs, *storePath);
     }

src/libflake/package.nix

@@ -1,19 +1,12 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 , nix-store
 , nix-fetchers
 , nix-expr
 , nlohmann_json
-, libgit2
-, man
 
 # Configuration Options
 
@@ -24,7 +17,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-flake";
   inherit version;
 
@@ -39,14 +32,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-store
     nix-util
@@ -67,10 +52,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libmain-c/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util-c
 , nix-store
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-main-c";
   inherit version;
 
@@ -38,14 +33,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util-c
     nix-store
@@ -68,10 +55,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libmain/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , openssl
 
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-main";
   inherit version;
 
@@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util
     nix-store
@@ -62,10 +49,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libstore-c/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util-c
 , nix-store
@@ -19,7 +14,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-store-c";
   inherit version;
 
@@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util-c
     nix-store
@@ -64,10 +51,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libstore/daemon.cc

@@ -199,6 +199,7 @@ struct ClientSettings
     time_t maxSilentTime;
     bool verboseBuild;
     unsigned int buildCores;
+    std::optional<unsigned int> loadLimit;
     bool useSubstitutes;
     StringMap overrides;
 
@@ -212,6 +213,7 @@ struct ClientSettings
         settings.maxSilentTime = maxSilentTime;
         settings.verboseBuild = verboseBuild;
         settings.buildCores = buildCores;
+        settings.loadLimit.assign(loadLimit);
         settings.useSubstitutes = useSubstitutes;
 
         for (auto & i : overrides) {

src/libstore/filetransfer.cc

@@ -759,12 +759,17 @@ struct curlFileTransfer : public FileTransfer
 
                 S3Helper s3Helper(profile, region, scheme, endpoint);
 
+                Activity act(*logger, lvlTalkative, actFileTransfer,
+                    fmt("downloading '%s'", request.uri),
+                    {request.uri}, request.parentAct);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;
                 if (!s3Res.data)
                     throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri);
                 res.data = std::move(*s3Res.data);
+                res.urls.push_back(request.uri);
                 callback(std::move(res));
 #else
                 throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);

src/libstore/globals.hh

@@ -161,7 +161,6 @@ public:
         R"(
           Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation.
           The `builder` executable can use this variable to control its own maximum amount of parallelism.
-
           <!--
           FIXME(@fricklerhandwerk): I don't think this should even be mentioned here.
           A very generic example using `derivation` and `xargs` may be more appropriate to explain the mechanism.
@@ -171,6 +170,8 @@ public:
 
           The value `0` means that the `builder` should use all available CPU cores in the system.
 
+          The [`load-limit`](#conf-load-limit) setting can be used to limit the total amount of build parallelism based on system load average.
+
           > **Note**
           >
           > The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting.
@@ -179,6 +180,33 @@ public:
         // Don't document the machine-specific default value
         false};
 
+    Setting<std::optional<unsigned int>> loadLimit{
+      this,
+      { getDefaultCores() },
+      "load-limit",
+      R"(
+        Sets the value of the `NIX_LOAD_LIMIT` environment variable in the
+        invocation of builders. Builders can use this value at their discretion
+        to dynamically control the amount of parallelism with respect to the
+        machine's load average.
+
+        For instance, a builder could use the value to set the `-l` flag to GNU
+        Make. In this case, if the load average of the machine exceeds
+        `NIX_LOAD_LIMIT`, the amount of parallelism will be dynamically
+        reduced.
+
+        By default, it is set to the number of cores on the machine.
+
+        On busy machines where Nix co-exists with other workloads, or where
+        build throughput is paramount and memory usage is not a bottleneck, the
+        default value may not work as intended. In this case, `load-limit`
+        should be set to a higher value, or to `none` to prevent the
+        `NIX_LOAD_LIMIT` variable being set at all.
+      )",
+      {},
+      // Don't document the machine-specific default value
+      false};
+
     /**
      * Read-only mode.  Don't copy stuff to the store, don't change
      * the database.

src/libstore/package.nix

@@ -1,11 +1,7 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
+, mkMesonLibrary
 
-, meson
-, ninja
-, pkg-config
 , unixtools
 
 , nix-util
@@ -29,7 +25,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-store";
   inherit version;
 
@@ -51,13 +47,8 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "sql") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ] ++ lib.optional embeddedSandboxShell unixtools.hexdump;
+  nativeBuildInputs =
+    lib.optional embeddedSandboxShell unixtools.hexdump;
 
   buildInputs = [
     boost
@@ -98,10 +89,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libstore/s3-binary-cache-store.cc

@@ -9,6 +9,7 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
+#include "signals.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -117,6 +118,7 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
 {
     bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
     {
+        checkInterrupt();
         auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
         if (retry)
             printError("AWS error '%s' (%s), will retry in %d ms",

src/libstore/unix/build/local-derivation-goal.cc

@@ -1202,6 +1202,10 @@ void LocalDerivationGoal::initEnv()
     /* The maximum number of cores to utilize for parallel building. */
     env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores);
 
+    /* Provide a load average limit for build tools to throttle jobs. */
+    if (settings.loadLimit.get())
+        env["NIX_LOAD_LIMIT"] = fmt("%d", settings.loadLimit.get().value());
+
     initTmpDir();
 
     /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
@@ -2008,7 +2012,7 @@ void LocalDerivationGoal::runChild()
             if (chdir(chrootRootDir.c_str()) == -1)
                 throw SysError("cannot change directory to '%1%'", chrootRootDir);
 
-            if (mkdir("real-root", 0) == -1)
+            if (mkdir("real-root", 0500) == -1)
                 throw SysError("cannot create real-root directory");
 
             if (pivot_root(".", "real-root") == -1)

src/libutil-c/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 
@@ -18,7 +13,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-util-c";
   inherit version;
 
@@ -35,14 +30,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util
   ];
@@ -62,10 +49,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libutil/config-impl.hh

@@ -107,6 +107,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 
 DECLARE_CONFIG_SERIALISER(std::string)
 DECLARE_CONFIG_SERIALISER(std::optional<std::string>)
+DECLARE_CONFIG_SERIALISER(std::optional<unsigned int>)
 DECLARE_CONFIG_SERIALISER(bool)
 DECLARE_CONFIG_SERIALISER(Strings)
 DECLARE_CONFIG_SERIALISER(StringSet)

src/libutil/config.cc

@@ -283,6 +283,25 @@ template<> std::string BaseSetting<std::optional<std::string>>::to_string() cons
     return value ? *value : "";
 }
 
+template<> std::optional<unsigned int> BaseSetting<std::optional<unsigned int>>::parse(const std::string & str) const
+{
+    if (str == "none") return std::nullopt;
+    else {
+        if (auto n = string2Int<unsigned int>(str))
+            return { *n };
+        else
+            throw UsageError("configuration setting '%s' should be 'none' or an integer", name);
+    }
+}
+
+template<> std::string BaseSetting<std::optional<unsigned int>>::to_string() const
+{
+    if (value)
+        return std::to_string(value.value());
+    else
+        return "none";
+}
+
 template<> bool BaseSetting<bool>::parse(const std::string & str) const
 {
     if (str == "true" || str == "yes" || str == "1")

src/libutil/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , boost
 , brotli
@@ -24,7 +19,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-util";
   inherit version;
 
@@ -43,14 +38,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     brotli
     libsodium
@@ -88,10 +75,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/nix-build/nix-build.cc

@@ -541,6 +541,8 @@ static void main_nix_build(int argc, char * * argv)
         env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
         env["NIX_STORE"] = store->storeDir;
         env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
+        if (settings.loadLimit.get())
+            env["NIX_LOAD_LIMIT"] = std::to_string(settings.loadLimit.get().value());
 
         auto passAsFile = tokenizeString<StringSet>(getOr(drv.env, "passAsFile", ""));
 

src/nix/eval.md

@@ -50,8 +50,9 @@ R""(
 
 # Description
 
-This command evaluates the given Nix expression and prints the
-result on standard output.
+This command evaluates the given Nix expression, and prints the result on standard output.
+
+It also evaluates any nested attribute values and list items.
 
 # Output format
 

src/nix/meson.build

@@ -212,18 +212,23 @@ nix_symlinks = [
   'nix-store',
 ]
 
+executable_suffix = ''
+if host_machine.system() == 'windows'
+  executable_suffix = '.exe'
+endif
+
 foreach linkname : nix_symlinks
   install_symlink(
-    linkname,
+    linkname + executable_suffix,
     # TODO(Qyriad): should these continue to be relative symlinks?
-    pointing_to : 'nix',
+    pointing_to : fs.name(this_exe),
     install_dir : get_option('bindir'),
     # The 'runtime' tag is what executables default to, which we want to emulate here.
     install_tag : 'runtime'
   )
   t = custom_target(
     command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
-    output: linkname,
+    output: linkname + executable_suffix,
     # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
     build_by_default: true
   )
@@ -233,15 +238,15 @@ endforeach
 
 install_symlink(
   'build-remote',
-  pointing_to : '..' / '..'/ get_option('bindir') / 'nix',
-  install_dir : get_option('libexecdir') / 'nix',
+  pointing_to : '..' / '..'/ get_option('bindir') / fs.name(this_exe),
+  install_dir : get_option('libexecdir') / fs.name(this_exe),
   # The 'runtime' tag is what executables default to, which we want to emulate here.
   install_tag : 'runtime'
 )
 
 custom_target(
   command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
-  output: 'build-remote',
+  output: 'build-remote' + executable_suffix,
   # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
   build_by_default: true
 )

src/nix/package.nix

@@ -1,21 +1,12 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-store
 , nix-expr
 , nix-main
 , nix-cmd
 
-, rapidcheck
-, gtest
-, runCommand
-
 # Configuration Options
 
 , version
@@ -25,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix";
   inherit version;
 
@@ -90,12 +81,6 @@ mkMesonDerivation (finalAttrs: {
     ]
   );
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-store
     nix-expr
@@ -118,10 +103,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/perl/package.nix

@@ -1,13 +1,10 @@
 { lib
 , stdenv
 , mkMesonDerivation
+, pkg-config
 , perl
 , perlPackages
-, meson
-, ninja
-, pkg-config
 , nix-store
-, darwin
 , version
 , curl
 , bzip2
@@ -36,8 +33,6 @@ perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: {
   ]);
 
   nativeBuildInputs = [
-    meson
-    ninja
     pkg-config
     perl
     curl

tests/functional/load-limit.nix

@@ -0,0 +1,8 @@
+with import ./config.nix;
+
+mkDerivation {
+  name = "load-limit";
+  buildCommand = ''
+    printf '%s' "''${NIX_LOAD_LIMIT-unset}" > "$out"
+  '';
+}

tests/functional/load-limit.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+TODO_NixOS
+
+clearStore
+
+outPath=$(nix-build --load-limit 100 load-limit.nix --no-out-link)
+text=$(cat "$outPath")
+if test "$text" != "100"; then exit 1; fi
+
+clearStore
+
+outPath=$(nix-build --load-limit 0 load-limit.nix --no-out-link)
+text=$(cat "$outPath")
+if test "$text" != "0"; then exit 1; fi
+
+clearStore
+
+outPath=$(nix-build --load-limit none load-limit.nix --no-out-link)
+text=$(cat "$outPath")
+if test "$text" != "unset"; then exit 1; fi

tests/functional/local.mk

@@ -95,6 +95,7 @@ nix_tests = \
   ssh-relay.sh \
   build.sh \
   build-delete.sh \
+  load-limit.sh \
   output-normalization.sh \
   selfref-gc.sh \
   db-migration.sh \

tests/functional/package.nix

@@ -75,7 +75,6 @@ mkMesonDerivation (finalAttrs: {
     nix-expr
   ];
 
-
   preConfigure =
     # "Inline" .version so it's not a symlink, and includes the suffix.
     # Do the meson utils, without modification.

tests/functional/tarball.sh

@@ -97,3 +97,17 @@ chmod +x "$TEST_ROOT/tar_root/foo"
 tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" .
 path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)"
 [[ $(cat "$path/foo") = bar ]]
+
+# Test a tarball with non-contiguous directory entries.
+rm -rf "$TEST_ROOT/tar_root"
+mkdir -p "$TEST_ROOT/tar_root/a/b"
+echo foo > "$TEST_ROOT/tar_root/a/b/foo"
+echo bla > "$TEST_ROOT/tar_root/bla"
+tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" .
+echo abc > "$TEST_ROOT/tar_root/bla"
+echo xyzzy > "$TEST_ROOT/tar_root/a/b/xyzzy"
+tar rvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" ./a/b/xyzzy ./bla
+path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)"
+[[ $(cat "$path/a/b/xyzzy") = xyzzy ]]
+[[ $(cat "$path/a/b/foo") = foo ]]
+[[ $(cat "$path/bla") = abc ]]

tests/nixos/s3-binary-cache-store.nix

@@ -51,6 +51,9 @@ in {
 
     server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}")
 
+    # Test fetchurl on s3:// URLs while we're at it.
+    client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000&region=eu-west-1\"; }'")
+
     # Copy a package from the binary cache.
     client.fail("nix path-info ${pkgA}")
 

tests/unit/libexpr-support/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-store-test-support
 , nix-expr
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-util-test-support";
   inherit version;
 
@@ -37,14 +32,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-store-test-support
     nix-expr
@@ -66,10 +53,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

tests/unit/libexpr/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-expr
 , nix-expr-c
@@ -26,7 +21,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-expr-tests";
   inherit version;
 
@@ -42,12 +37,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-expr
     nix-expr-c
@@ -71,10 +60,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = runCommand "${finalAttrs.pname}-run" {

tests/unit/libfetchers/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-fetchers
 , nix-store-test-support
@@ -25,7 +20,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-fetchers-tests";
   inherit version;
 
@@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-fetchers
     nix-store-test-support
@@ -69,10 +58,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = runCommand "${finalAttrs.pname}-run" {

tests/unit/libflake/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-flake
 , nix-expr-test-support
@@ -25,7 +20,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-flake-tests";
   inherit version;
 
@@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-flake
     nix-expr-test-support
@@ -69,10 +58,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = runCommand "${finalAttrs.pname}-run" {

tests/unit/libstore-support/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util-test-support
 , nix-store
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-store-test-support";
   inherit version;
 
@@ -37,14 +32,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util-test-support
     nix-store
@@ -66,10 +53,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

tests/unit/libstore/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-store
 , nix-store-c
@@ -27,7 +22,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-store-tests";
   inherit version;
 
@@ -43,12 +38,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-store
     nix-store-c
@@ -73,10 +62,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = let

tests/unit/libutil-support/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 
@@ -20,7 +15,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-util-test-support";
   inherit version;
 
@@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util
     rapidcheck
@@ -64,10 +51,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

tests/unit/libutil/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-util
 , nix-util-c
@@ -25,7 +20,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-util-tests";
   inherit version;
 
@@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-util
     nix-util-c
@@ -70,10 +59,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = runCommand "${finalAttrs.pname}-run" {