Merge 21ea502f78 into 806a91f7bf

Clean up the `package.nix` files

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v29
+    - uses: cachix/install-nix-action@v30
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -89,7 +89,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v29
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - uses: cachix/cachix-action@v15
@@ -112,7 +112,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v29
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -142,7 +142,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v29
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV

doc/manual/.version

@@ -0,0 +1 @@
+../../.version

doc/manual/book.toml

@@ -7,9 +7,21 @@ additional-js = ["redirects.js"]
 edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
 git-repository-url = "https://github.com/NixOS/nix"
 
+# Handles replacing @docroot@ with a path to ./src relative to that markdown file,
+# {{#include handlebars}}, and the @generated@ syntax used within these. it mostly
+# but not entirely replaces the links preprocessor (which we cannot simply use due
+# to @generated@ files living in a different directory to make meson happy). we do
+# not want to disable the links preprocessor entirely though because that requires
+# disabling *all* built-in preprocessors and selectively reenabling those we want.
+[preprocessor.substitute]
+command = "python3 ./substitute.py"
+before = ["anchors", "links"]
+
 [preprocessor.anchors]
 renderers = ["html"]
-command = "jq --from-file doc/manual/anchors.jq"
+command = "jq --from-file ./anchors.jq"
+
+[output.markdown]
 
 [output.linkcheck]
 # no Internet during the build (in the sandbox)

doc/manual/generate-deps.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import glob
+import sys
+
+# meson expects makefile-style dependency declarations, i.e.
+#
+#   target: dependency...
+#
+# meson seems to pass depfiles straight on to ninja even though
+# it also parses the file itself (or at least has code to do so
+# in its tree), so we must live by ninja's rules: only slashes,
+# spaces and octothorpes can be escaped, anything else is taken
+# literally. since the rules for these aren't even the same for
+# all three we will just fail when we encounter any of them (if
+# asserts are off for some reason the depfile will likely point
+# to nonexistant paths, making everything phony and thus fine.)
+for path in glob.glob(sys.argv[1] + '/**', recursive=True):
+    assert '\\' not in path
+    assert ' ' not in path
+    assert '#' not in path
+    print("ignored:", path)

doc/manual/local.mk

@@ -223,8 +223,13 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/
 			sed -i "s,@docroot@,$$docroot,g" "$$file"; \
 		done; \
 		set -euo pipefail; \
-		RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
-			| { grep -Fv "because fragment resolution isn't implemented" || :; }; \
+		( \
+		    cd "$$tmp/manual"; \
+		    RUST_LOG=warn \
+		        MDBOOK_SUBSTITUTE_SEARCH=$(d)/src \
+		        mdbook build -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
+			    | { grep -Fv "because fragment resolution isn't implemented" || :; } \
+		); \
 		rm -rf "$$tmp/manual"
 	@rm -rf $(DESTDIR)$(docdir)/manual
 	@mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual

doc/manual/meson.build

@@ -0,0 +1,353 @@
+project('nix-manual',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+nix = find_program('nix', native : true)
+
+mdbook = find_program('mdbook', native : true)
+bash = find_program('bash', native : true)
+
+pymod = import('python')
+python = pymod.find_installation('python3')
+
+nix_env_for_docs = {
+  'HOME': '/dummy',
+  'NIX_CONF_DIR': '/dummy',
+  'NIX_SSL_CERT_FILE': '/dummy/no-ca-bundle.crt',
+  'NIX_STATE_DIR': '/dummy',
+  'NIX_CONFIG': 'cores = 0',
+}
+
+nix_for_docs = [nix, '--experimental-features', 'nix-command']
+nix_eval_for_docs_common = nix_for_docs + [
+  'eval',
+  '-I', 'nix=' + meson.current_source_dir(),
+  '--store', 'dummy://',
+  '--impure',
+]
+nix_eval_for_docs = nix_eval_for_docs_common + '--raw'
+
+conf_file_json = custom_target(
+  command : nix_for_docs + ['config', 'show', '--json'],
+  capture : true,
+  output : 'conf-file.json',
+  env : nix_env_for_docs,
+)
+
+language_json = custom_target(
+  command: [nix, '__dump-language'],
+  output : 'language.json',
+  capture : true,
+  env : nix_env_for_docs,
+)
+
+nix3_cli_json = custom_target(
+  command : [nix, '__dump-cli'],
+  capture : true,
+  output : 'nix.json',
+  env : nix_env_for_docs,
+)
+
+generate_manual_deps = files(
+  'generate-deps.py',
+)
+
+# Generates types
+subdir('src/store')
+# Generates builtins.md and builtin-constants.md.
+subdir('src/language')
+# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md.
+subdir('src/command-ref')
+# Generates experimental-feature-descriptions.md.
+subdir('src/development')
+# Generates rl-next-generated.md.
+subdir('src/release-notes')
+subdir('src')
+
+# Hacky way to figure out if `nix` is an `ExternalProgram` or
+# `Exectuable`. Only the latter can occur in custom target input lists.
+if nix.full_path().startswith(meson.build_root())
+  nix_input = nix
+else
+  nix_input = []
+endif
+
+manual = custom_target(
+  'manual',
+  command : [
+    bash,
+    '-euo', 'pipefail',
+    '-c',
+    '''
+        @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
+        @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/src/SUMMARY.md.in > @2@/src/SUMMARY.md
+        rsync -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
+        (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
+        rm -rf @2@/manual
+        mv @2@/html @2@/manual
+        find @2@/manual -iname meson.build -delete
+    '''.format(
+      python.full_path(),
+      mdbook.full_path(),
+      meson.current_build_dir(),
+    ),
+  ],
+  input : [
+    generate_manual_deps,
+    'substitute.py',
+    'book.toml',
+    'anchors.jq',
+    'custom.css',
+    nix3_cli_files,
+    experimental_features_shortlist_md,
+    experimental_feature_descriptions_md,
+    types_dir,
+    conf_file_md,
+    builtins_md,
+    rl_next_generated,
+    summary_rl_next,
+    nix_input,
+  ],
+  output : [
+    'manual',
+    'markdown',
+  ],
+  depfile : 'manual.d',
+  env : {
+    'RUST_LOG': 'info',
+    'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src',
+  },
+)
+manual_html = manual[0]
+manual_md = manual[1]
+
+install_subdir(
+  manual_html.full_path(),
+  install_dir : get_option('datadir') / 'doc/nix',
+)
+
+nix_nested_manpages = [
+  [ 'nix-env',
+    [
+      'delete-generations',
+      'install',
+      'list-generations',
+      'query',
+      'rollback',
+      'set-flag',
+      'set',
+      'switch-generation',
+      'switch-profile',
+      'uninstall',
+      'upgrade',
+    ],
+  ],
+  [ 'nix-store',
+    [
+      'add-fixed',
+      'add',
+      'delete',
+      'dump-db',
+      'dump',
+      'export',
+      'gc',
+      'generate-binary-cache-key',
+      'import',
+      'load-db',
+      'optimise',
+      'print-env',
+      'query',
+      'read-log',
+      'realise',
+      'repair-path',
+      'restore',
+      'serve',
+      'verify',
+      'verify-path',
+    ],
+  ],
+]
+
+foreach command : nix_nested_manpages
+  foreach page : command[1]
+    title = command[0] + ' --' + page
+    section = '1'
+    custom_target(
+      command : [
+        bash,
+        files('./render-manpage.sh'),
+        '--out-no-smarty',
+        title,
+        section,
+        '@INPUT0@/command-ref' / command[0] / (page + '.md'),
+        '@OUTPUT0@',
+      ],
+      input : [
+        manual_md,
+        nix_input,
+      ],
+      output : command[0] + '-' + page + '.1',
+      install : true,
+      install_dir : get_option('mandir') / 'man1',
+    )
+  endforeach
+endforeach
+
+nix3_manpages = [
+  'nix3-build',
+  'nix3-bundle',
+  'nix3-config',
+  'nix3-config-show',
+  'nix3-copy',
+  'nix3-daemon',
+  'nix3-derivation-add',
+  'nix3-derivation',
+  'nix3-derivation-show',
+  'nix3-develop',
+  #'nix3-doctor',
+  'nix3-edit',
+  'nix3-eval',
+  'nix3-flake-archive',
+  'nix3-flake-check',
+  'nix3-flake-clone',
+  'nix3-flake-info',
+  'nix3-flake-init',
+  'nix3-flake-lock',
+  'nix3-flake',
+  'nix3-flake-metadata',
+  'nix3-flake-new',
+  'nix3-flake-prefetch',
+  'nix3-flake-show',
+  'nix3-flake-update',
+  'nix3-fmt',
+  'nix3-hash-file',
+  'nix3-hash',
+  'nix3-hash-path',
+  'nix3-hash-to-base16',
+  'nix3-hash-to-base32',
+  'nix3-hash-to-base64',
+  'nix3-hash-to-sri',
+  'nix3-help',
+  'nix3-help-stores',
+  'nix3-key-convert-secret-to-public',
+  'nix3-key-generate-secret',
+  'nix3-key',
+  'nix3-log',
+  'nix3-nar-cat',
+  'nix3-nar-dump-path',
+  'nix3-nar-ls',
+  'nix3-nar',
+  'nix3-path-info',
+  'nix3-print-dev-env',
+  'nix3-profile-diff-closures',
+  'nix3-profile-history',
+  'nix3-profile-install',
+  'nix3-profile-list',
+  'nix3-profile',
+  'nix3-profile-remove',
+  'nix3-profile-rollback',
+  'nix3-profile-upgrade',
+  'nix3-profile-wipe-history',
+  'nix3-realisation-info',
+  'nix3-realisation',
+  'nix3-registry-add',
+  'nix3-registry-list',
+  'nix3-registry',
+  'nix3-registry-pin',
+  'nix3-registry-remove',
+  'nix3-repl',
+  'nix3-run',
+  'nix3-search',
+  #'nix3-shell',
+  'nix3-store-add-file',
+  'nix3-store-add-path',
+  'nix3-store-cat',
+  'nix3-store-copy-log',
+  'nix3-store-copy-sigs',
+  'nix3-store-delete',
+  'nix3-store-diff-closures',
+  'nix3-store-dump-path',
+  'nix3-store-gc',
+  'nix3-store-ls',
+  'nix3-store-make-content-addressed',
+  'nix3-store',
+  'nix3-store-optimise',
+  'nix3-store-path-from-hash-part',
+  'nix3-store-ping',
+  'nix3-store-prefetch-file',
+  'nix3-store-repair',
+  'nix3-store-sign',
+  'nix3-store-verify',
+  'nix3-upgrade-nix',
+  'nix3-why-depends',
+  'nix',
+]
+
+foreach page : nix3_manpages
+  section = '1'
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      page,
+      section,
+      '@INPUT1@/command-ref/new-cli/@0@.md'.format(page),
+      '@OUTPUT@',
+    ],
+    input : [
+      files('./render-manpage.sh'),
+      manual_md,
+      nix_input,
+    ],
+    output : page + '.1',
+    install : true,
+    install_dir : get_option('mandir') / 'man1',
+  )
+endforeach
+
+nix_manpages = [
+  [ 'nix-env', 1 ],
+  [ 'nix-store', 1 ],
+  [ 'nix-build', 1 ],
+  [ 'nix-shell', 1 ],
+  [ 'nix-instantiate', 1 ],
+  [ 'nix-collect-garbage', 1 ],
+  [ 'nix-prefetch-url', 1 ],
+  [ 'nix-channel', 1 ],
+  [ 'nix-hash', 1 ],
+  [ 'nix-copy-closure', 1 ],
+  [ 'nix.conf', 5, conf_file_md.full_path() ],
+  [ 'nix-daemon', 8 ],
+  [ 'nix-profiles', 5, 'files/profiles.md' ],
+]
+
+foreach entry : nix_manpages
+  title = entry[0]
+  # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md,
+  # rather than a stem identical to its mdbook source.
+  # Therefore we use an optional third element of this array to override the name pattern
+  md_file = entry.get(2, title + '.md')
+  section = entry[1].to_string()
+  md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file)
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      title,
+      section,
+      md_file_resolved,
+      '@OUTPUT@',
+    ],
+    input : [
+      files('./render-manpage.sh'),
+      manual_md,
+      entry.get(3, []),
+      nix_input,
+    ],
+    output : '@0@.@1@'.format(entry[0], entry[1]),
+    install : true,
+    install_dir : get_option('mandir') / 'man@0@'.format(entry[1]),
+  )
+endforeach

doc/manual/package.nix

@@ -0,0 +1,71 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, lowdown
+, mdbook
+, mdbook-linkcheck
+, jq
+, python3
+, rsync
+, nix-cli
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-manual";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.difference
+    (fileset.unions [
+      ../../.version
+      # Too many different types of files to filter for now
+      ../../doc/manual
+      ./.
+    ])
+    # Do a blacklist instead
+    ../../doc/manual/package.nix;
+
+  # TODO the man pages should probably be separate
+  outputs = [ "out" "man" ];
+
+  # Hack for sake of the dev shell
+  passthru.externalNativeBuildInputs = [
+    meson
+    ninja
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+    jq
+    python3
+    rsync
+  ];
+
+  nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [
+    nix-cli
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''$out/nix-support
+    echo "doc manual ''$out/share/doc/nix/manual" >> ''$out/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

doc/manual/remove_before_wrapper.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import shutil
+import typing as t
+
+def main():
+    if len(sys.argv) < 4 or '--' not in sys.argv:
+        print("Usage: remove-before-wrapper <output> -- <nix command...>")
+        sys.exit(1)
+
+    # Extract the parts
+    output: str = sys.argv[1]
+    nix_command_idx: int = sys.argv.index('--') + 1
+    nix_command: t.List[str] = sys.argv[nix_command_idx:]
+
+    output_temp: str = output + '.tmp'
+
+    # Remove the output and temp output in case they exist
+    shutil.rmtree(output, ignore_errors=True)
+    shutil.rmtree(output_temp, ignore_errors=True)
+
+    # Execute nix command with `--write-to` tempary output
+    nix_command_write_to = nix_command + ['--write-to', output_temp]
+    subprocess.run(nix_command_write_to, check=True)
+
+    # Move the temporary output to the intended location
+    os.rename(output_temp, output)
+
+if __name__ == "__main__":
+    main()

doc/manual/render-manpage.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+lowdown_args=
+
+if [ "$1" = --out-no-smarty ]; then
+    lowdown_args=--out-no-smarty
+    shift
+fi
+
+[ "$#" = 4 ] || {
+    echo "wrong number of args passed" >&2
+    exit 1
+}
+
+title="$1"
+section="$2"
+infile="$3"
+outfile="$4"
+
+(
+    printf "Title: %s\n\n" "$title"
+    cat "$infile"
+) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile"

doc/manual/rl-next/build-hook-default.md

@@ -16,7 +16,7 @@ This has a small adverse affect on remote building --- the `build-remote` execut
 This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work.
 
 Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221).
-There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder.
+There should simply be no need to have an extra, intermediate layer of remote-procedure-calling when we want to connect to a remote builder.
 The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/index.md) i.e. `Store` subclass. <!-- we normally don't mention classes, but consider that this release note is about a library use case -->
 
-The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above.
+The Perl bindings no longer expose `getBinDir` either, since the underlying C++ libraries those bindings wrap no longer know the location of installed binaries as described above.

doc/manual/src/advanced-topics/distributed-builds.md

@@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can
 test whether connecting to the remote Nix instance works, e.g.
 
 ```console
-$ nix store ping --store ssh://mac
+$ nix store info --store ssh://mac
 ```
 
 will try to connect to the machine named `mac`. It is possible to
 specify an SSH identity file as part of the remote store URI, e.g.
 
 ```console
-$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key
+$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key
 ```
 
 Since builds should be non-interactive, the key should not have a

doc/manual/src/command-ref/files/channels.md

@@ -1,26 +1,41 @@
 ## Channels
 
-A directory containing symlinks to Nix channels, managed by [`nix-channel`]:
+Channels are a mechanism for obtaining Nix expressions over the web.
+
+The moving parts of channels are:
+- The official channels listed at <https://nixos.org/channels>
+- The user-specific list of [subscribed channels](#subscribed-channels)
+- The [downloaded channel contents](#channel-contents)
+- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path)
+
+> **Note**
+>
+> The state of a subscribed channel is external to the Nix expressions relying on it.
+> This may limit reproducibility.
+>
+> Dependencies on other Nix expressions can be declared explicitly with:
+> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions
+> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations for explicitly modifying the value of [lookup paths](@docroot@/language/constructs/lookup-path.md)
+
+## Subscribed channels
+
+The list of subscribed channels is stored in a file:
+
+- `~/.nix-channels`
+- `$XDG_STATE_HOME/nix/channels` if [`use-xdg-base-directories`](@docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories) is set to `true`
+
+Each line maps a channel name to a URL in the following format:
+
+```
+<url> <name>
+```
+
+## Channels contents
+
+The [`nix-channel`](@docroot@/command-ref/nix-channel.md) command uses a [profile](@docroot@/command-ref/files/profiles.md) to keep track of channels:
 
 - `$XDG_STATE_HOME/nix/profiles/channels` for regular users
 - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root`
 
-[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels.
-This profile contains symlinks to the contents of those channels.
-
-## Subscribed channels
-
-The list of subscribed channels is stored in
-
-- `~/.nix-channels`
-- `$XDG_STATE_HOME/nix/channels` if [`use-xdg-base-directories`] is set to `true`
-
-in the following format:
-
-```
-<url> <name>
-...
-```
-
-[`nix-channel`]: @docroot@/command-ref/nix-channel.md
-[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories
+Each generation of that profile is a directory with symlinks to channel contents.
+Each entry in this directory corresponds to the name of a [subscribed channel](#subscribed-channels) at that time.

doc/manual/src/command-ref/meson.build

@@ -0,0 +1,63 @@
+xp_features_json = custom_target(
+  command : [nix, '__dump-xp-features'],
+  capture : true,
+  output : 'xp-features.json',
+)
+
+experimental_features_shortlist_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile ./@INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features-shortlist.nix',
+    xp_features_json,
+  ],
+  output : 'experimental-features-shortlist.md',
+  capture : true,
+  env : nix_env_for_docs,
+)
+
+nix3_cli_files = custom_target(
+  command : [
+    python.full_path(),
+    '@INPUT0@',
+    '@OUTPUT@',
+    '--'
+  ] + nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT1@ true (builtins.readFile ./@INPUT2@)',
+  ],
+  input : [
+    '../../remove_before_wrapper.py',
+    '../../generate-manpage.nix',
+    nix3_cli_json,
+  ],
+  output : 'new-cli',
+  env : nix_env_for_docs,
+)
+
+conf_file_md_body = custom_target(
+  command : [
+    nix_eval_for_docs,
+    '--expr',
+    'import @INPUT0@ { prefix = "conf"; } (builtins.fromJSON (builtins.readFile ./@INPUT1@))',
+  ],
+  capture : true,
+  input : [
+    '../../generate-settings.nix',
+    conf_file_json,
+  ],
+  output : 'conf-file.body.md',
+  env : nix_env_for_docs,
+)
+
+conf_file_md = custom_target(
+  command : [ 'cat', '@INPUT0@', '@INPUT1@' ],
+  capture : true,
+  input : [
+    'conf-file-prefix.md',
+    conf_file_md_body,
+  ],
+  output : 'conf-file.md',
+)

doc/manual/src/command-ref/nix-channel.md

@@ -1,6 +1,6 @@
 # Name
 
-`nix-channel` - manage Nix channels
+`nix-channel` - manage [channels](#channels)
 
 # Synopsis
 
@@ -8,24 +8,12 @@
 
 # Description
 
-Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version.
+This command allows
+- Managing a list of [subscribed channels](#subscribed-channels)
+- Retrieval of the [contents](#channel-contents) of subscribed channels
+- Restoring the contents of channels to previously retrieved versions
 
-The moving parts of channels are:
-- The official channels listed at <https://nixos.org/channels>
-- The user-specific list of [subscribed channels](#subscribed-channels)
-- The [downloaded channel contents](#channels)
-- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH)
-
-> **Note**
->
-> The state of a subscribed channel is external to the Nix expressions relying on it.
-> This may limit reproducibility.
->
-> Dependencies on other Nix expressions can be declared explicitly with:
-> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions
-> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations
-
-This command has the following operations:
+It has the following operations:
 
 - `--add` *url* \[*name*\]
 

doc/manual/src/command-ref/opt-common.md

@@ -1,3 +1,7 @@
+<!-- Some of the options documented here are hardcopied from
+     src/libcmd/common-eval-args.cc
+-->
+
 # Common Options
 
 Most Nix commands accept the following command-line options:
@@ -161,6 +165,14 @@ Most Nix commands accept the following command-line options:
   You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`.
   (Note that since the argument is a Nix string literal, you have to escape the quotes.)
 
+- <span id="opt-arg-from-file">[`--arg-from-file`](#opt-arg-from-file)</span> *name* *path*
+
+  Pass the contents of file *path* as the argument *name* to Nix functions.
+
+- <span id="opt-arg-from-stdin">[`--arg-from-stdin`](#opt-arg-from-stdin)</span> *name*
+
+  Pass the contents of stdin as the argument *name* to Nix functions.
+
 - <span id="opt-argstr">[`--argstr`](#opt-argstr)</span> *name* *value*
 
   This option is like `--arg`, only the value is not a Nix expression but a string.
@@ -179,6 +191,10 @@ Most Nix commands accept the following command-line options:
   attribute of the fourth element of the array in the `foo` attribute
   of the top-level expression.
 
+- <span id="opt-eval-store">[`--eval-store`](#opt-eval-store)</span> *store-url*
+
+  The [URL to the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. where to store derivations (`.drv` files) and inputs referenced by them.
+
 - <span id="opt-expr">[`--expr`](#opt-expr)</span> / `-E`
 
   Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions.
@@ -194,6 +210,10 @@ Most Nix commands accept the following command-line options:
 
   Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH).
 
+- <span id="opt-impure">[`--impure`](#opt-impure)</span>
+
+  Allow access to mutable paths and repositories.
+
 - <span id="opt-option">[`--option`](#opt-option)</span> *name* *value*
 
   Set the Nix configuration option *name* to *value*.

doc/manual/src/development/meson.build

@@ -0,0 +1,12 @@
+experimental_feature_descriptions_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features.nix',
+    xp_features_json,
+  ],
+  capture : true,
+  output : 'experimental-feature-descriptions.md',
+)

doc/manual/src/installation/uninstall.md

@@ -100,7 +100,7 @@ which you may remove.
    LABEL=Nix\040Store /nix apfs rw,nobrowse
    ```
 
-   by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file.
+   by setting the cursor on the respective line using the arrow keys, and pressing `dd`, and then `:wq` to save the file.
 
    This will prevent automatic mounting of the Nix Store volume.
 

doc/manual/src/language/constructs.md

@@ -0,0 +1 @@
+# Language Constructs

doc/manual/src/language/meson.build

@@ -0,0 +1,20 @@
+builtins_md = custom_target(
+  command : [
+    python.full_path(),
+    '@INPUT0@',
+    '@OUTPUT@',
+    '--'
+  ] + nix_eval_for_docs + [
+    '--expr',
+    '(builtins.readFile @INPUT3@) + import @INPUT1@ (builtins.fromJSON (builtins.readFile ./@INPUT2@)) + (builtins.readFile @INPUT4@)',
+  ],
+  input : [
+    '../../remove_before_wrapper.py',
+    '../../generate-builtins.nix',
+    language_json,
+    'builtins-prefix.md',
+    'builtins-suffix.md'
+  ],
+  output : 'builtins.md',
+  env : nix_env_for_docs,
+)

doc/manual/src/language/values.md

@@ -0,0 +1 @@
+# Data Types

doc/manual/src/meson.build

@@ -0,0 +1,17 @@
+summary_rl_next = custom_target(
+  command : [
+    bash,
+    '-euo', 'pipefail',
+    '-c',
+    '''
+      if [ -e "@INPUT@" ]; then
+        echo '  - [Upcoming release](release-notes/rl-next.md)'
+      fi
+    ''',
+  ],
+  input : [
+    rl_next_generated,
+  ],
+  capture: true,
+  output : 'SUMMARY-rl-next.md',
+)

doc/manual/src/protocols/json/index.md

@@ -0,0 +1 @@
+# JSON Formats

doc/manual/src/release-notes/meson.build

@@ -0,0 +1,24 @@
+rl_next_generated = custom_target(
+  command : [
+    'bash',
+    '-euo',
+    'pipefail',
+    '-c',
+    '''
+      if type -p build-release-notes > /dev/null; then
+        build-release-notes --change-authors @CURRENT_SOURCE_DIR@/../../change-authors.yml @CURRENT_SOURCE_DIR@/../../rl-next
+      elif type -p changelog-d > /dev/null; then
+        changelog-d @CURRENT_SOURCE_DIR@/../../rl-next
+      fi
+      @0@ @INPUT0@ @CURRENT_SOURCE_DIR@/../../rl-next > @DEPFILE@
+    '''.format(
+      python.full_path(),
+    ),
+  ],
+  input : [
+    generate_manual_deps,
+  ],
+  output : 'rl-next.md',
+  capture : true,
+  depfile : 'rl-next.d',
+)

doc/manual/src/store/meson.build

@@ -0,0 +1,18 @@
+types_dir = custom_target(
+  command : [
+    python.full_path(),
+    '@INPUT0@',
+    '@OUTPUT@',
+    '--'
+  ] + nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT1@ (builtins.fromJSON (builtins.readFile ./@INPUT2@)).stores',
+  ],
+  input : [
+    '../../remove_before_wrapper.py',
+    '../../generate-store-types.nix',
+    nix3_cli_json,
+  ],
+  output : 'types',
+  env : nix_env_for_docs,
+)

doc/manual/substitute.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import json
+import os, os.path
+import sys
+import typing as t
+
+name = 'substitute.py'
+
+def log(*args: t.Any, **kwargs: t.Any) -> None:
+    kwargs['file'] = sys.stderr
+    print(f'{name}:', *args, **kwargs)
+
+def do_include(content: str, relative_md_path: Path, source_root: Path, search_path: Path) -> str:
+    assert not relative_md_path.is_absolute(), f'{relative_md_path=} from mdbook should be relative'
+
+    md_path_abs = source_root / relative_md_path
+    var_abs = md_path_abs.parent
+    assert var_abs.is_dir(), f'supposed directory {var_abs} is not a directory (cwd={os.getcwd()})'
+
+    lines = []
+    for l in content.splitlines(keepends=True):
+        if l.strip().startswith("{{#include "):
+            requested = l.strip()[11:][:-2]
+            if requested.startswith("@generated@/"):
+                included = search_path / Path(requested[12:])
+                requested = included.relative_to(search_path)
+            else:
+                included = source_root / relative_md_path.parent / requested
+                requested = included.resolve().relative_to(source_root)
+            assert included.exists(), f"{requested} not found at {included}"
+            lines.append(do_include(included.read_text(), requested, source_root, search_path) + "\n")
+        else:
+            lines.append(l)
+    return "".join(lines)
+
+def recursive_replace(data: dict[str, t.Any], book_root: Path, search_path: Path) -> dict[str, t.Any]:
+    match data:
+        case {'sections': sections}:
+            return data | dict(
+                sections = [recursive_replace(section, book_root, search_path) for section in sections],
+            )
+        case {'Chapter': chapter}:
+            path_to_chapter = Path(chapter['path'])
+            chapter_content = chapter['content']
+
+            return data | dict(
+                Chapter = chapter | dict(
+                    # first process includes. this must happen before docroot processing since
+                    # mdbook does not see these included files, only the final agglomeration.
+                    content = do_include(
+                        chapter_content,
+                        path_to_chapter,
+                        book_root,
+                        search_path
+                    ).replace(
+                        '@docroot@',
+                        ("../" * len(path_to_chapter.parent.parts) or "./")[:-1]
+                    ),
+                    sub_items = [
+                        recursive_replace(sub_item, book_root, search_path)
+                        for sub_item in chapter['sub_items']
+                    ],
+                ),
+            )
+
+        case rest:
+            assert False, f'should have been called on a dict, not {type(rest)=}\n\t{rest=}'
+
+def main() -> None:
+
+
+    if len(sys.argv) > 1 and sys.argv[1] == 'supports':
+        return 0
+
+    # includes pointing into @generated@ will look here
+    search_path = Path(os.environ['MDBOOK_SUBSTITUTE_SEARCH'])
+
+    if len(sys.argv) > 1 and sys.argv[1] == 'summary':
+        print(do_include(
+                        sys.stdin.read(),
+                        Path('src/SUMMARY.md'),
+                        Path(sys.argv[2]).resolve(),
+                        search_path))
+        return
+
+    # mdbook communicates with us over stdin and stdout.
+    # It splorks us a JSON array, the first element describing the context,
+    # the second element describing the book itself,
+    # and then expects us to send it the modified book JSON over stdout.
+
+    context, book = json.load(sys.stdin)
+
+    # book_root is the directory where book contents leave (ie, src/)
+    book_root = Path(context['root']) / context['config']['book']['src']
+
+    # Find @var@ in all parts of our recursive book structure.
+    replaced_content = recursive_replace(book, book_root, search_path)
+
+    replaced_content_str = json.dumps(replaced_content)
+
+    # Give mdbook our changes.
+    print(replaced_content_str)
+
+try:
+    sys.exit(main())
+except AssertionError as e:
+    print(f'{name}: INTERNAL ERROR in mdbook preprocessor: {e}', file=sys.stderr)
+    print(f'this is a bug in {name}', file=sys.stderr)
+    raise

flake.nix

@@ -221,6 +221,7 @@
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
           default = self.packages.${system}.nix-ng;
+          nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual;
           nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
           nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
         }
@@ -349,9 +350,10 @@
             ++ pkgs.nixComponents.nix-store.nativeBuildInputs
             ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
             ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
+            ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs
             ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
             ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
-            ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs
+            ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs
             ++ lib.optional
               (!buildCanExecuteHost
                  # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479

meson.build

@@ -23,6 +23,9 @@ subproject('nix')
 # Docs
 subproject('internal-api-docs')
 subproject('external-api-docs')
+if not meson.is_cross_build()
+  subproject('nix-manual')
+endif
 
 # External C wrapper libraries
 subproject('libutil-c')

packaging/components.nix

@@ -60,6 +60,7 @@ in
 
   nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; };
 
+  nix-manual = callPackage ../doc/manual/package.nix { version = fineVersion; };
   nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; };
   nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; };
 

packaging/dependencies.nix

@@ -38,6 +38,10 @@ let
   # Indirection for Nixpkgs to override when package.nix files are vendored
   filesetToSource = lib.fileset.toSource;
 
+  /** Given a set of layers, create a mkDerivation-like function */
+  mkPackageBuilder = exts: userFn:
+    stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn);
+
   localSourceLayer = finalAttrs: prevAttrs:
     let
       workDirPath =
@@ -60,6 +64,28 @@ let
       workDir = null;
     };
 
+  mesonLayer = finalAttrs: prevAttrs:
+    {
+      nativeBuildInputs = [
+        pkgs.buildPackages.meson
+        pkgs.buildPackages.ninja
+      ] ++ prevAttrs.nativeBuildInputs or [];
+    };
+
+  mesonBuildLayer = finalAttrs: prevAttrs:
+    {
+      nativeBuildInputs = prevAttrs.nativeBuildInputs or [] ++ [
+        pkgs.buildPackages.pkg-config
+      ];
+      separateDebugInfo = !stdenv.hostPlatform.isStatic;
+      hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+    };
+
+  mesonLibraryLayer = finalAttrs: prevAttrs:
+    {
+      outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
+    };
+
   # Work around weird `--as-needed` linker behavior with BSD, see
   # https://github.com/mesonbuild/meson/issues/3593
   bsdNoLinkAsNeeded = finalAttrs: prevAttrs:
@@ -172,14 +198,27 @@ scope: {
 
   inherit resolvePath filesetToSource;
 
-  mkMesonDerivation = f: let
-    exts = [
+  mkMesonDerivation =
+    mkPackageBuilder [
+      miscGoodPractice
+      localSourceLayer
+      mesonLayer
+    ];
+  mkMesonExecutable =
+    mkPackageBuilder [
       miscGoodPractice
       bsdNoLinkAsNeeded
       localSourceLayer
+      mesonLayer
+      mesonBuildLayer
+    ];
+  mkMesonLibrary =
+    mkPackageBuilder [
+      miscGoodPractice
+      bsdNoLinkAsNeeded
+      localSourceLayer
+      mesonLayer
+      mesonBuildLayer
+      mesonLibraryLayer
     ];
-  in stdenv.mkDerivation
-   (lib.extends
-     (lib.foldr lib.composeExtensions (_: _: {}) exts)
-     f);
 }

packaging/everything.nix

@@ -33,6 +33,7 @@
 
   nix-functional-tests,
 
+  nix-manual,
   nix-internal-api-docs,
   nix-external-api-docs,
 
@@ -70,6 +71,7 @@
 
     nix-cli
 
+    nix-manual
     nix-internal-api-docs
     nix-external-api-docs
 

packaging/hydra.nix

@@ -146,6 +146,9 @@ in
     withCoverageChecks = true;
   };
 
+  # Nix's manual
+  manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual;
+
   # API docs for Nix's unstable internal C++ interfaces.
   internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs;
 

scripts/meson.build

@@ -1,5 +1,3 @@
-# configures `scripts/nix-profile.sh.in` (and copies the original to the build directory).
-# this is only needed for tests, but running it unconditionally does not hurt enough to care.
 configure_file(
   input : 'nix-profile.sh.in',
   output : 'nix-profile.sh',
@@ -8,13 +6,6 @@ configure_file(
   }
 )
 
-# https://github.com/mesonbuild/meson/issues/860
-configure_file(
-  input : 'nix-profile.sh.in',
-  output : 'nix-profile.sh.in',
-  copy : true,
-)
-
 foreach rc : [ '.sh', '.fish', '-daemon.sh', '-daemon.fish' ]
   configure_file(
     input : 'nix-profile' + rc  + '.in',

src/external-api-docs/package.nix

@@ -1,8 +1,6 @@
 { lib
 , mkMesonDerivation
 
-, meson
-, ninja
 , doxygen
 
 # Configuration Options
@@ -37,8 +35,6 @@ mkMesonDerivation (finalAttrs: {
     ];
 
   nativeBuildInputs = [
-    meson
-    ninja
     doxygen
   ];
 

src/internal-api-docs/package.nix

@@ -1,8 +1,6 @@
 { lib
 , mkMesonDerivation
 
-, meson
-, ninja
 , doxygen
 
 # Configuration Options
@@ -32,8 +30,6 @@ mkMesonDerivation (finalAttrs: {
   ];
 
   nativeBuildInputs = [
-    meson
-    ninja
     doxygen
   ];
 

src/libcmd/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 , nix-store
@@ -38,7 +33,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-cmd";
   inherit version;
 
@@ -54,14 +49,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     ({ inherit editline readline; }.${readlineFlavor})
   ] ++ lib.optional enableMarkdown lowdown;
@@ -93,10 +80,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libcmd/repl.cc

@@ -645,7 +645,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
         } else if (fallbackPos) {
-            std::stringstream ss;
+            std::ostringstream ss;
             ss << "Attribute `" << fallbackName << "`\n\n";
             ss << "  … defined at " << state->positions[fallbackPos] << "\n\n";
             if (fallbackDoc) {
@@ -654,7 +654,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 ss << "No documentation found.\n\n";
             }
 
-            auto markdown = ss.str();
+            auto markdown = toView(ss);
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
 
         } else
@@ -826,7 +826,7 @@ void NixRepl::runNix(Path program, const Strings & args, const std::optional<std
     if (runNixPtr)
         (*runNixPtr)(program, args, input);
     else
-        throw Error("Cannot run '%s', no method of calling the Nix CLI provided", program);
+        throw Error("Cannot run '%s' because no method of calling the Nix CLI was provided. This is a configuration problem pertaining to how this program was built. See Nix 2.25 release notes", program);
 }
 
 

src/libexpr-c/package.nix

@@ -1,10 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-store-c
 , nix-expr
@@ -18,7 +14,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-expr-c";
   inherit version;
 
@@ -35,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-store-c
     nix-expr
@@ -63,10 +51,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libexpr/eval.cc

@@ -539,7 +539,7 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
     if (v.isLambda()) {
         auto exprLambda = v.payload.lambda.fun;
 
-        std::stringstream s(std::ios_base::out);
+        std::ostringstream s;
         std::string name;
         auto pos = positions[exprLambda->getPos()];
         std::string docStr;
@@ -571,17 +571,12 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 
         s << docStr;
 
-        s << '\0'; // for making a c string below
-        std::string ss = s.str();
-
         return Doc {
             .pos = pos,
             .name = name,
             .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
             .args = {},
-            .doc =
-                // FIXME: this leaks; make the field std::string?
-                strdup(ss.data()),
+            .doc = makeImmutableString(toView(s)), // NOTE: memory leak when compiled without GC
         };
     }
     if (isFunctor(v)) {
@@ -1805,11 +1800,9 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v)
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
     if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
-        auto exprStr = ({
-            std::ostringstream out;
-            cond->show(state.symbols, out);
-            out.str();
-        });
+        std::ostringstream out;
+        cond->show(state.symbols, out);
+        auto exprStr = toView(out);
 
         if (auto eq = dynamic_cast<ExprOpEq *>(cond)) {
             try {

src/libexpr/get-drvs.cc

@@ -374,11 +374,12 @@ static void getDerivations(EvalState & state, Value & vIn,
            bound to the attribute with the "lower" name should take
            precedence). */
         for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
+            std::string_view symbol{state.symbols[i->name]};
             try {
-                debug("evaluating attribute '%1%'", state.symbols[i->name]);
-                if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
+                debug("evaluating attribute '%1%'", symbol);
+                if (!std::regex_match(symbol.begin(), symbol.end(), attrRegex))
                     continue;
-                std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
+                std::string pathPrefix2 = addToPath(pathPrefix, symbol);
                 if (combineChannels)
                     getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
@@ -392,7 +393,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                     }
                 }
             } catch (Error & e) {
-                e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]);
+                e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", symbol);
                 throw;
             }
         }

src/libexpr/package.nix

@@ -1,11 +1,7 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
+, mkMesonLibrary
 
-, meson
-, ninja
-, pkg-config
 , bison
 , flex
 , cmake # for resolving toml11 dep
@@ -38,7 +34,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-expr";
   inherit version;
 
@@ -55,15 +51,13 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
     ./lexer.l
     ./parser.y
-    (fileset.fileFilter (file: file.hasExt "nix") ./.)
+    (fileset.difference
+      (fileset.fileFilter (file: file.hasExt "nix") ./.)
+      ./package.nix
+    )
   ];
 
-  outputs = [ "out" "dev" ];
-
   nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
     bison
     flex
     cmake
@@ -102,10 +96,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libexpr/primops.cc

@@ -40,6 +40,13 @@ namespace nix {
  * Miscellaneous
  *************************************************************/
 
+static inline Value * mkString(EvalState & state, const std::csub_match & match)
+{
+    Value * v = state.allocValue();
+    v->mkString({match.first, match.second});
+    return v;
+}
+
 StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD)
 {
     std::vector<DerivedPath::Built> drvs;
@@ -2129,7 +2136,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Valu
     std::ostringstream out;
     NixStringContext context;
     printValueAsXML(state, true, false, *args[0], out, context, pos);
-    v.mkString(out.str(), context);
+    v.mkString(toView(out), context);
 }
 
 static RegisterPrimOp primop_toXML({
@@ -2237,7 +2244,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Val
     std::ostringstream out;
     NixStringContext context;
     printValueAsJSON(state, true, *args[0], pos, out, context);
-    v.mkString(out.str(), context);
+    v.mkString(toView(out), context);
 }
 
 static RegisterPrimOp primop_toJSON({
@@ -4268,7 +4275,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
             if (!match[i + 1].matched)
                 v2 = &state.vNull;
             else
-                (v2 = state.allocValue())->mkString(match[i + 1].str());
+                v2 = mkString(state, match[i + 1]);
         v.mkList(list);
 
     } catch (std::regex_error & e) {
@@ -4352,7 +4359,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
             auto match = *i;
 
             // Add a string for non-matched characters.
-            (list[idx++] = state.allocValue())->mkString(match.prefix().str());
+            list[idx++] = mkString(state, match.prefix());
 
             // Add a list for matched substrings.
             const size_t slen = match.size() - 1;
@@ -4363,14 +4370,14 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
                 if (!match[si + 1].matched)
                     v2 = &state.vNull;
                 else
-                    (v2 = state.allocValue())->mkString(match[si + 1].str());
+                    v2 = mkString(state, match[si + 1]);
             }
 
             (list[idx++] = state.allocValue())->mkList(list2);
 
             // Add a string for non-matched suffix characters.
             if (idx == 2 * len)
-                (list[idx++] = state.allocValue())->mkString(match.suffix().str());
+                list[idx++] = mkString(state, match.suffix());
         }
 
         assert(idx == 2 * len + 1);

src/libexpr/primops/fetchTree.cc

@@ -11,6 +11,8 @@
 #include "value-to-json.hh"
 #include "fetch-to-store.hh"
 
+#include <nlohmann/json.hpp>
+
 #include <ctime>
 #include <iomanip>
 #include <regex>

src/libexpr/primops/fromTOML.cc

@@ -66,7 +66,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V
                         attrs.alloc("_type").mkString("timestamp");
                         std::ostringstream s;
                         s << t;
-                        attrs.alloc("value").mkString(s.str());
+                        attrs.alloc("value").mkString(toView(s));
                         v.mkAttrs(attrs);
                     } else {
                         throw std::runtime_error("Dates and times are not supported");

src/libexpr/print.cc

@@ -460,7 +460,7 @@ private:
 
                 std::ostringstream s;
                 s << state.positions[v.payload.lambda.fun->pos];
-                output << " @ " << filterANSIEscapes(s.str());
+                output << " @ " << filterANSIEscapes(toView(s));
             }
         } else if (v.isPrimOp()) {
             if (v.primOp())

src/libfetchers/git-utils.cc

@@ -208,7 +208,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
 static void initRepoAtomically(std::filesystem::path &path, bool bare) {
     if (pathExists(path.string())) return;
 
-    Path tmpDir = createTempDir(std::filesystem::path(path).parent_path());
+    Path tmpDir = createTempDir(os_string_to_string(PathViewNG { std::filesystem::path(path).parent_path() }));
     AutoDelete delTmpDir(tmpDir, true);
     Repository tmpRepo;
 
@@ -977,8 +977,24 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
     void pushBuilder(std::string name)
     {
+        const git_tree_entry * entry;
+        Tree prevTree = nullptr;
+
+        if (!pendingDirs.empty() &&
+            (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str())))
+        {
+            /* Clone a tree that we've already finished. This happens
+               if a tarball has directory entries that are not
+               contiguous. */
+            if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+                throw Error("parent of '%s' is not a directory", name);
+
+            if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(prevTree), *repo, entry))
+                throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
+        }
+
         git_treebuilder * b;
-        if (git_treebuilder_new(&b, *repo, nullptr))
+        if (git_treebuilder_new(&b, *repo, prevTree.get()))
             throw Error("creating a tree builder: %s", git_error_last()->message);
         pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) });
     };

src/libfetchers/git.cc

@@ -13,8 +13,8 @@
 #include "git-utils.hh"
 #include "logging.hh"
 #include "finally.hh"
-
 #include "fetch-settings.hh"
+#include "json-utils.hh"
 
 #include <regex>
 #include <string.h>

src/libfetchers/package.nix

@@ -1,17 +1,11 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 , nix-store
 , nlohmann_json
 , libgit2
-, man
 
 # Configuration Options
 
@@ -22,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-fetchers";
   inherit version;
 
@@ -37,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     libgit2
   ];
@@ -67,10 +53,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libfetchers/tarball.cc

@@ -90,6 +90,7 @@ DownloadFileResult downloadFile(
     /* Cache metadata for all URLs in the redirect chain. */
     for (auto & url : res.urls) {
         key.second.insert_or_assign("url", url);
+        assert(!res.urls.empty());
         infoAttrs.insert_or_assign("url", *res.urls.rbegin());
         getCache()->upsert(key, *store, infoAttrs, *storePath);
     }

src/libflake/flake/flake.cc

@@ -13,6 +13,8 @@
 #include "value-to-json.hh"
 #include "local-fs-store.hh"
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 using namespace flake;

src/libflake/package.nix

@@ -1,19 +1,12 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 , nix-store
 , nix-fetchers
 , nix-expr
 , nlohmann_json
-, libgit2
-, man
 
 # Configuration Options
 
@@ -24,7 +17,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-flake";
   inherit version;
 
@@ -39,14 +32,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-store
     nix-util
@@ -67,10 +52,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libmain-c/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util-c
 , nix-store
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-main-c";
   inherit version;
 
@@ -38,14 +33,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util-c
     nix-store
@@ -68,10 +55,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libmain/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , openssl
 
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-main";
   inherit version;
 
@@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util
     nix-store
@@ -62,10 +49,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libmain/progress-bar.cc

@@ -158,10 +158,10 @@ public:
     {
         auto state(state_.lock());
 
-        std::stringstream oss;
+        std::ostringstream oss;
         showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
-        log(*state, ei.level, oss.str());
+        log(*state, ei.level, toView(oss));
     }
 
     void log(State & state, Verbosity lvl, std::string_view s)

src/libstore-c/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util-c
 , nix-store
@@ -19,7 +14,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-store-c";
   inherit version;
 
@@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util-c
     nix-store
@@ -64,10 +51,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libstore/builtins.hh

@@ -9,7 +9,8 @@ namespace nix {
 void builtinFetchurl(
     const BasicDerivation & drv,
     const std::map<std::string, Path> & outputs,
-    const std::string & netrcData);
+    const std::string & netrcData,
+    const std::string & caFileData);
 
 void builtinUnpackChannel(
     const BasicDerivation & drv,

src/libstore/builtins/fetchurl.cc

@@ -9,7 +9,8 @@ namespace nix {
 void builtinFetchurl(
     const BasicDerivation & drv,
     const std::map<std::string, Path> & outputs,
-    const std::string & netrcData)
+    const std::string & netrcData,
+    const std::string & caFileData)
 {
     /* Make the host's netrc data available. Too bad curl requires
        this to be stored in a file. It would be nice if we could just
@@ -19,6 +20,9 @@ void builtinFetchurl(
         writeFile(settings.netrcFile, netrcData, 0600);
     }
 
+    settings.caFile = "ca-certificates.crt";
+    writeFile(settings.caFile, caFileData, 0600);
+
     auto out = get(drv.outputs, "out");
     if (!out)
         throw Error("'builtin:fetchurl' requires an 'out' output");

src/libstore/daemon.cc

@@ -90,11 +90,11 @@ struct TunnelLogger : public Logger
     {
         if (ei.level > verbosity) return;
 
-        std::stringstream oss;
+        std::ostringstream oss;
         showErrorInfo(oss, ei, false);
 
         StringSink buf;
-        buf << STDERR_NEXT << oss.str();
+        buf << STDERR_NEXT << toView(oss);
         enqueueMsg(buf.s);
     }
 

src/libstore/derivations.cc

@@ -7,11 +7,12 @@
 #include "split.hh"
 #include "common-protocol.hh"
 #include "common-protocol-impl.hh"
+#include "strings-inline.hh"
+#include "json-utils.hh"
+
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
 
-#include "strings-inline.hh"
-
 namespace nix {
 
 std::optional<StorePath> DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const

src/libstore/filetransfer.cc

@@ -759,12 +759,17 @@ struct curlFileTransfer : public FileTransfer
 
                 S3Helper s3Helper(profile, region, scheme, endpoint);
 
+                Activity act(*logger, lvlTalkative, actFileTransfer,
+                    fmt("downloading '%s'", request.uri),
+                    {request.uri}, request.parentAct);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;
                 if (!s3Res.data)
                     throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri);
                 res.data = std::move(*s3Res.data);
+                res.urls.push_back(request.uri);
                 callback(std::move(res));
 #else
                 throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);

src/libstore/local-store.cc

@@ -50,6 +50,8 @@
 
 #include <sqlite3.h>
 
+#include <nlohmann/json.hpp>
+
 #include "strings.hh"
 
 

src/libstore/nar-info.cc

@@ -2,6 +2,7 @@
 #include "nar-info.hh"
 #include "store-api.hh"
 #include "strings.hh"
+#include "json-utils.hh"
 
 namespace nix {
 

src/libstore/outputs-spec.cc

@@ -30,16 +30,15 @@ std::optional<OutputsSpec> OutputsSpec::parseOpt(std::string_view s)
 {
     static std::regex regex(std::string { outputSpecRegexStr });
 
-    std::smatch match;
-    std::string s2 { s }; // until some improves std::regex
-    if (!std::regex_match(s2, match, regex))
+    std::cmatch match;
+    if (!std::regex_match(s.cbegin(), s.cend(), match, regex))
         return std::nullopt;
 
     if (match[1].matched)
         return { OutputsSpec::All {} };
 
     if (match[2].matched)
-        return OutputsSpec::Names { tokenizeString<StringSet>(match[2].str(), ",") };
+        return OutputsSpec::Names { tokenizeString<StringSet>({match[2].first, match[2].second}, ",") };
 
     assert(false);
 }

src/libstore/package.nix

@@ -1,11 +1,7 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
+, mkMesonLibrary
 
-, meson
-, ninja
-, pkg-config
 , unixtools
 
 , nix-util
@@ -29,7 +25,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-store";
   inherit version;
 
@@ -51,13 +47,8 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "sql") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ] ++ lib.optional embeddedSandboxShell unixtools.hexdump;
+  nativeBuildInputs =
+    lib.optional embeddedSandboxShell unixtools.hexdump;
 
   buildInputs = [
     boost
@@ -98,10 +89,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libstore/s3-binary-cache-store.cc

@@ -9,6 +9,7 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
+#include "signals.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -117,6 +118,7 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
 {
     bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
     {
+        checkInterrupt();
         auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
         if (retry)
             printError("AWS error '%s' (%s), will retry in %d ms",

src/libstore/unix/build/local-derivation-goal.cc

@@ -433,6 +433,41 @@ static void doBind(const Path & source, const Path & target, bool optional = fal
 };
 #endif
 
+/**
+ * Rethrow the current exception as a subclass of `Error`.
+ */
+static void rethrowExceptionAsError()
+{
+    try {
+        throw;
+    } catch (Error &) {
+        throw;
+    } catch (std::exception & e) {
+        throw Error(e.what());
+    } catch (...) {
+        throw Error("unknown exception");
+    }
+}
+
+/**
+ * Send the current exception to the parent in the format expected by
+ * `LocalDerivationGoal::processSandboxSetupMessages()`.
+ */
+static void handleChildException(bool sendException)
+{
+    try {
+        rethrowExceptionAsError();
+    } catch (Error & e) {
+        if (sendException) {
+            writeFull(STDERR_FILENO, "\1\n");
+            FdSink sink(STDERR_FILENO);
+            sink << e;
+            sink.flush();
+        } else
+            std::cerr << e.msg();
+    }
+}
+
 void LocalDerivationGoal::startBuilder()
 {
     if ((buildUser && buildUser->getUIDCount() != 1)
@@ -949,32 +984,40 @@ void LocalDerivationGoal::startBuilder()
                root. */
             openSlave();
 
-            /* Drop additional groups here because we can't do it
-               after we've created the new user namespace. */
-            if (setgroups(0, 0) == -1) {
-                if (errno != EPERM)
-                    throw SysError("setgroups failed");
-                if (settings.requireDropSupplementaryGroups)
-                    throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step.");
+            try {
+                /* Drop additional groups here because we can't do it
+                   after we've created the new user namespace. */
+                if (setgroups(0, 0) == -1) {
+                    if (errno != EPERM)
+                        throw SysError("setgroups failed");
+                    if (settings.requireDropSupplementaryGroups)
+                        throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step.");
+                }
+
+                ProcessOptions options;
+                options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
+                if (privateNetwork)
+                    options.cloneFlags |= CLONE_NEWNET;
+                if (usingUserNamespace)
+                    options.cloneFlags |= CLONE_NEWUSER;
+
+                pid_t child = startProcess([&]() { runChild(); }, options);
+
+                writeFull(sendPid.writeSide.get(), fmt("%d\n", child));
+                _exit(0);
+            } catch (...) {
+                handleChildException(true);
+                _exit(1);
             }
-
-            ProcessOptions options;
-            options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
-            if (privateNetwork)
-                options.cloneFlags |= CLONE_NEWNET;
-            if (usingUserNamespace)
-                options.cloneFlags |= CLONE_NEWUSER;
-
-            pid_t child = startProcess([&]() { runChild(); }, options);
-
-            writeFull(sendPid.writeSide.get(), fmt("%d\n", child));
-            _exit(0);
         });
 
         sendPid.writeSide.close();
 
-        if (helper.wait() != 0)
+        if (helper.wait() != 0) {
+            processSandboxSetupMessages();
+            // Only reached if the child process didn't send an exception.
             throw Error("unable to start build process");
+        }
 
         userNamespaceSync.readSide = -1;
 
@@ -1050,7 +1093,12 @@ void LocalDerivationGoal::startBuilder()
     pid.setSeparatePG(true);
     worker.childStarted(shared_from_this(), {builderOut.get()}, true, true);
 
-    /* Check if setting up the build environment failed. */
+    processSandboxSetupMessages();
+}
+
+
+void LocalDerivationGoal::processSandboxSetupMessages()
+{
     std::vector<std::string> msgs;
     while (true) {
         std::string msg = [&]() {
@@ -1078,7 +1126,8 @@ void LocalDerivationGoal::startBuilder()
 }
 
 
-void LocalDerivationGoal::initTmpDir() {
+void LocalDerivationGoal::initTmpDir()
+{
     /* In a sandbox, for determinism, always use the same temporary
        directory. */
 #if __linux__
@@ -1743,13 +1792,20 @@ void LocalDerivationGoal::runChild()
 
         bool setUser = true;
 
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
+        /* Make the contents of netrc and the CA certificate bundle
+           available to builtin:fetchurl (which may run under a
+           different uid and/or in a sandbox). */
         std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SystemError &) { }
+        std::string caFileData;
+        if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") {
+           try {
+               netrcData = readFile(settings.netrcFile);
+           } catch (SystemError &) { }
+
+           try {
+               caFileData = readFile(settings.caFile);
+           } catch (SystemError &) { }
+        }
 
 #if __linux__
         if (useChroot) {
@@ -1952,7 +2008,7 @@ void LocalDerivationGoal::runChild()
             if (chdir(chrootRootDir.c_str()) == -1)
                 throw SysError("cannot change directory to '%1%'", chrootRootDir);
 
-            if (mkdir("real-root", 0) == -1)
+            if (mkdir("real-root", 0500) == -1)
                 throw SysError("cannot create real-root directory");
 
             if (pivot_root(".", "real-root") == -1)
@@ -2188,7 +2244,7 @@ void LocalDerivationGoal::runChild()
                         worker.store.printStorePath(scratchOutputs.at(e.first)));
 
                 if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(*drv, outputs, netrcData);
+                    builtinFetchurl(*drv, outputs, netrcData, caFileData);
                 else if (drv->builder == "builtin:buildenv")
                     builtinBuildenv(*drv, outputs);
                 else if (drv->builder == "builtin:unpack-channel")
@@ -2230,14 +2286,8 @@ void LocalDerivationGoal::runChild()
 
         throw SysError("executing '%1%'", drv->builder);
 
-    } catch (Error & e) {
-        if (sendException) {
-            writeFull(STDERR_FILENO, "\1\n");
-            FdSink sink(STDERR_FILENO);
-            sink << e;
-            sink.flush();
-        } else
-            std::cerr << e.msg();
+    } catch (...) {
+        handleChildException(sendException);
         _exit(1);
     }
 }

src/libstore/unix/build/local-derivation-goal.hh

@@ -210,6 +210,11 @@ struct LocalDerivationGoal : public DerivationGoal
      */
     void initEnv();
 
+    /**
+     * Process messages send by the sandbox initialization.
+     */
+    void processSandboxSetupMessages();
+
     /**
      * Setup tmp dir location.
      */

src/libutil-c/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util
 
@@ -18,7 +13,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-util-c";
   inherit version;
 
@@ -35,14 +30,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "h") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util
   ];
@@ -62,10 +49,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libutil/args.cc

@@ -293,7 +293,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
                     // We match one space after `nix` so that we preserve indentation.
                     // No space is necessary for an empty line. An empty line has basically no effect.
                     if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$")))
-                        shebangContent += match[2].str() + "\n";
+                        shebangContent += std::string_view{match[2].first, match[2].second} + "\n";
                 }
                 for (const auto & word : parseShebangContent(shebangContent)) {
                     cmdline.push_back(word);

src/libutil/config-global.cc

@@ -1,5 +1,7 @@
 #include "config-global.hh"
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 bool GlobalConfig::set(const std::string & name, const std::string & value)

src/libutil/experimental-features.hh

@@ -2,9 +2,10 @@
 ///@file
 
 #include "error.hh"
-#include "json-utils.hh"
 #include "types.hh"
 
+#include <nlohmann/json_fwd.hpp>
+
 namespace nix {
 
 /**
@@ -98,10 +99,4 @@ public:
 void to_json(nlohmann::json &, const ExperimentalFeature &);
 void from_json(const nlohmann::json &, ExperimentalFeature &);
 
-/**
- * It is always rendered as a string
- */
-template<>
-struct json_avoids_null<ExperimentalFeature> : std::true_type {};
-
 }

src/libutil/json-utils.hh

@@ -3,12 +3,13 @@
 
 #include <nlohmann/json.hpp>
 #include <list>
-#include <nlohmann/json_fwd.hpp>
 
 #include "types.hh"
 
 namespace nix {
 
+enum struct ExperimentalFeature;
+
 const nlohmann::json * get(const nlohmann::json & map, const std::string & key);
 
 nlohmann::json * get(nlohmann::json & map, const std::string & key);
@@ -71,6 +72,12 @@ struct json_avoids_null<std::list<T>> : std::true_type {};
 template<typename K, typename V>
 struct json_avoids_null<std::map<K, V>> : std::true_type {};
 
+/**
+ * `ExperimentalFeature` is always rendered as a string.
+ */
+template<>
+struct json_avoids_null<ExperimentalFeature> : std::true_type {};
+
 }
 
 namespace nlohmann {

src/libutil/logging.cc

@@ -85,10 +85,10 @@ public:
 
     void logEI(const ErrorInfo & ei) override
     {
-        std::stringstream oss;
+        std::ostringstream oss;
         showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
-        log(ei.level, oss.str());
+        log(ei.level, toView(oss));
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

src/libutil/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , boost
 , brotli
@@ -24,7 +19,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-util";
   inherit version;
 
@@ -43,14 +38,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     brotli
     libsodium
@@ -88,10 +75,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libutil/strings.cc

@@ -6,6 +6,21 @@
 
 namespace nix {
 
+struct view_stringbuf : public std::stringbuf
+{
+    inline std::string_view toView()
+    {
+        auto begin = pbase();
+        return {begin, begin + pubseekoff(0, std::ios_base::cur, std::ios_base::out)};
+    }
+};
+
+std::string_view toView(const std::ostringstream & os)
+{
+    auto buf = static_cast<view_stringbuf *>(os.rdbuf());
+    return buf->toView();
+}
+
 template std::list<std::string> tokenizeString(std::string_view s, std::string_view separators);
 template std::set<std::string> tokenizeString(std::string_view s, std::string_view separators);
 template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);

src/libutil/strings.hh

@@ -8,6 +8,11 @@
 
 namespace nix {
 
+/*
+ * workaround for unavailable view() method (C++20) of std::ostringstream under MacOS with clang-16
+ */
+std::string_view toView(const std::ostringstream & os);
+
 /**
  * String tokenizer.
  *

src/nix-build/nix-build.cc

@@ -36,7 +36,7 @@ extern char * * environ __attribute__((weak));
 /* Recreate the effect of the perl shellwords function, breaking up a
  * string into arguments like a shell word, including escapes
  */
-static std::vector<std::string> shellwords(const std::string & s)
+static std::vector<std::string> shellwords(std::string_view s)
 {
     std::regex whitespace("^\\s+");
     auto begin = s.cbegin();
@@ -51,7 +51,7 @@ static std::vector<std::string> shellwords(const std::string & s)
     auto it = begin;
     for (; it != s.cend(); ++it) {
         if (st == sBegin) {
-            std::smatch match;
+            std::cmatch match;
             if (regex_search(it, s.cend(), match, whitespace)) {
                 cur.append(begin, it);
                 res.push_back(cur);
@@ -173,7 +173,7 @@ static void main_nix_build(int argc, char * * argv)
                     line = chomp(line);
                     std::smatch match;
                     if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$")))
-                        for (const auto & word : shellwords(match[1].str()))
+                        for (const auto & word : shellwords({match[1].first, match[1].second}))
                             args.push_back(word);
                 }
             }
@@ -260,9 +260,9 @@ static void main_nix_build(int argc, char * * argv)
                 // read the shebang to understand which packages to read from. Since
                 // this is handled via nix-shell -p, we wrap our ruby script execution
                 // in ruby -e 'load' which ignores the shebangs.
-                envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), joined.str());
+                envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined));
             } else {
-                envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), joined.str());
+                envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined));
             }
         }
 

src/nix-env/user-env.cc

@@ -111,9 +111,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems,
     auto manifestFile = ({
         std::ostringstream str;
         printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits<int>::max());
-        // TODO with C++20 we can use str.view() instead and avoid copy.
-        std::string str2 = str.str();
-        StringSource source { str2 };
+        StringSource source { toView(str) };
         state.store->addToStoreFromDump(
             source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references);
     });

src/nix-manual

@@ -0,0 +1 @@
+../doc/manual/

src/nix/config-check.cc

@@ -26,17 +26,17 @@ std::string formatProtocol(unsigned int proto)
     return "unknown";
 }
 
-bool checkPass(const std::string & msg) {
+bool checkPass(std::string_view msg) {
     notice(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg);
     return true;
 }
 
-bool checkFail(const std::string & msg) {
+bool checkFail(std::string_view msg) {
     notice(ANSI_RED "[FAIL] " ANSI_NORMAL + msg);
     return false;
 }
 
-void checkInfo(const std::string & msg) {
+void checkInfo(std::string_view msg) {
     notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg);
 }
 
@@ -87,11 +87,11 @@ struct CmdConfigCheck : StoreCommand
         }
 
         if (dirs.size() != 1) {
-            std::stringstream ss;
+            std::ostringstream ss;
             ss << "Multiple versions of nix found in PATH:\n";
             for (auto & dir : dirs)
                 ss << "  " << dir << "\n";
-            return checkFail(ss.str());
+            return checkFail(toView(ss));
         }
 
         return checkPass("PATH contains only one nix version.");
@@ -125,14 +125,14 @@ struct CmdConfigCheck : StoreCommand
         }
 
         if (!dirs.empty()) {
-            std::stringstream ss;
+            std::ostringstream ss;
             ss << "Found profiles outside of " << settings.nixStateDir << "/profiles.\n"
                << "The generation this profile points to might not have a gcroot and could be\n"
                << "garbage collected, resulting in broken symlinks.\n\n";
             for (auto & dir : dirs)
                 ss << "  " << dir << "\n";
             ss << "\n";
-            return checkFail(ss.str());
+            return checkFail(toView(ss));
         }
 
         return checkPass("All profiles are gcroots.");
@@ -145,13 +145,13 @@ struct CmdConfigCheck : StoreCommand
             : PROTOCOL_VERSION;
 
         if (clientProto != storeProto) {
-            std::stringstream ss;
+            std::ostringstream ss;
             ss << "Warning: protocol version of this client does not match the store.\n"
                << "While this is not necessarily a problem it's recommended to keep the client in\n"
                << "sync with the daemon.\n\n"
                << "Client protocol: " << formatProtocol(clientProto) << "\n"
                << "Store protocol: " << formatProtocol(storeProto) << "\n\n";
-            return checkFail(ss.str());
+            return checkFail(toView(ss));
         }
 
         return checkPass("Client protocol matches store protocol.");

src/nix/eval.cc

@@ -11,11 +11,13 @@
 
 using namespace nix;
 
+namespace nix::fs { using namespace std::filesystem; }
+
 struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
 {
     bool raw = false;
     std::optional<std::string> apply;
-    std::optional<Path> writeTo;
+    std::optional<fs::path> writeTo;
 
     CmdEval() : InstallableValueCommand()
     {
@@ -75,20 +77,20 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
         if (writeTo) {
             stopProgressBar();
 
-            if (pathExists(*writeTo))
-                throw Error("path '%s' already exists", *writeTo);
+            if (fs::symlink_exists(*writeTo))
+                throw Error("path '%s' already exists", writeTo->string());
 
-            std::function<void(Value & v, const PosIdx pos, const std::filesystem::path & path)> recurse;
+            std::function<void(Value & v, const PosIdx pos, const fs::path & path)> recurse;
 
-            recurse = [&](Value & v, const PosIdx pos, const std::filesystem::path & path)
+            recurse = [&](Value & v, const PosIdx pos, const fs::path & path)
             {
                 state->forceValue(v, pos);
                 if (v.type() == nString)
                     // FIXME: disallow strings with contexts?
                     writeFile(path.string(), v.string_view());
                 else if (v.type() == nAttrs) {
-                    // TODO abstract mkdir perms for Windows
-                    createDir(path.string(), 0777);
+                    // Directory should not already exist
+                    assert(fs::create_directory(path.string()));
                     for (auto & attr : *v.attrs()) {
                         std::string_view name = state->symbols[attr.name];
                         try {

src/nix/eval.md

@@ -50,8 +50,9 @@ R""(
 
 # Description
 
-This command evaluates the given Nix expression and prints the
-result on standard output.
+This command evaluates the given Nix expression, and prints the result on standard output.
+
+It also evaluates any nested attribute values and list items.
 
 # Output format
 

src/nix/main.cc

@@ -19,6 +19,7 @@
 #include "eval-cache.hh"
 #include "flake/flake.hh"
 #include "self-exe.hh"
+#include "json-utils.hh"
 
 #include <sys/types.h>
 #include <regex>

src/nix/meson.build

@@ -212,18 +212,23 @@ nix_symlinks = [
   'nix-store',
 ]
 
+executable_suffix = ''
+if host_machine.system() == 'windows'
+  executable_suffix = '.exe'
+endif
+
 foreach linkname : nix_symlinks
   install_symlink(
-    linkname,
+    linkname + executable_suffix,
     # TODO(Qyriad): should these continue to be relative symlinks?
-    pointing_to : 'nix',
+    pointing_to : fs.name(this_exe),
     install_dir : get_option('bindir'),
     # The 'runtime' tag is what executables default to, which we want to emulate here.
     install_tag : 'runtime'
   )
   t = custom_target(
     command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
-    output: linkname,
+    output: linkname + executable_suffix,
     # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
     build_by_default: true
   )
@@ -233,15 +238,15 @@ endforeach
 
 install_symlink(
   'build-remote',
-  pointing_to : '..' / '..'/ get_option('bindir') / 'nix',
-  install_dir : get_option('libexecdir') / 'nix',
+  pointing_to : '..' / '..'/ get_option('bindir') / fs.name(this_exe),
+  install_dir : get_option('libexecdir') / fs.name(this_exe),
   # The 'runtime' tag is what executables default to, which we want to emulate here.
   install_tag : 'runtime'
 )
 
 custom_target(
   command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'],
-  output: 'build-remote',
+  output: 'build-remote' + executable_suffix,
   # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
   build_by_default: true
 )

src/nix/package.nix

@@ -1,21 +1,12 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-store
 , nix-expr
 , nix-main
 , nix-cmd
 
-, rapidcheck
-, gtest
-, runCommand
-
 # Configuration Options
 
 , version
@@ -25,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix";
   inherit version;
 
@@ -90,12 +81,6 @@ mkMesonDerivation (finalAttrs: {
     ]
   );
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-store
     nix-expr
@@ -118,10 +103,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/nix/self-exe.cc

@@ -13,11 +13,12 @@ fs::path getNixBin(std::optional<std::string_view> binaryNameOpt)
 {
     auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; };
 
-    // If the environment variable is set, use it unconditionally
+    // If the environment variable is set, use it unconditionally.
     if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR"))
         return fs::path{*envOpt} / std::string{getBinaryName()};
 
-    // Use some-times avaiable OS tricks to get to the path of this Nix, and try that
+    // Try OS tricks, if available, to get to the path of this Nix, and
+    // see if we can find the right executable next to that.
     if (auto selfOpt = getSelfExe()) {
         fs::path path{*selfOpt};
         if (binaryNameOpt)

src/nix/self-exe.hh

@@ -17,9 +17,9 @@ namespace nix {
  * Instead, we'll query the OS for the path to the current executable,
  * using `getSelfExe()`.
  *
- * As a last resort, we resort to `PATH`. Hopefully we find a `nix`
- * there that's compatible. If you're porting Nix to a new platform,
- * that might be good enough for a while, but you'll want to improve
+ * As a last resort, we rely on `PATH`. Hopefully we find a `nix` there
+ * that's compatible. If you're porting Nix to a new platform, that
+ * might be good enough for a while, but you'll want to improve
  * `getSelfExe()` to work on your platform.
  *
  * @param binary_name the exact binary name we're looking up. Might be

src/perl/package.nix

@@ -1,13 +1,10 @@
 { lib
 , stdenv
 , mkMesonDerivation
+, pkg-config
 , perl
 , perlPackages
-, meson
-, ninja
-, pkg-config
 , nix-store
-, darwin
 , version
 , curl
 , bzip2
@@ -36,8 +33,6 @@ perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: {
   ]);
 
   nativeBuildInputs = [
-    meson
-    ninja
     pkg-config
     perl
     curl

tests/functional/meson.build

@@ -253,8 +253,9 @@ foreach suite : suites
         'NIX_REMOTE': '',
         'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ',
       },
-      # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine
-      # this can easily drive them to failure. give them more time than default of 30sec
+      # Some tests take 15+ seconds even on an otherwise idle machine;
+      # on a loaded machine this can easily drive them to failure. Give
+      # them more time than the default of 30 seconds.
       timeout : 300,
       # Used for target dependency/ordering tracking, not adding compiler flags or anything.
       depends : suite['deps'],

tests/functional/nars.sh

@@ -88,17 +88,50 @@ touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3"
 rm -rf "$TEST_ROOT/case"
 expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case-collision.nar | grepQuiet "NAR contains file name 'test' that collides with case-hacked file name 'Test~nix~case~hack~1'"
 
-# Deserializing a NAR that contains file names that Unicode-normalize
-# to the same name should fail on macOS but succeed on Linux.
+# Deserializing a NAR that contains file names that Unicode-normalize to the
+# same name should fail on macOS and specific Linux setups (typically ZFS with
+# `utf8only` enabled and `normalization` set to anything else than `none`). The
+# deserialization should succeed on most Linux, where file names aren't
+# unicode-normalized.
+#
+# We test that:
+#
+# 1. It either succeeds or fails with "already exists" error.
+# 2. Nix has the same behavior with respect to unicode normalization than
+#    $TEST_ROOT's filesystem (when using basic Unix commands)
 rm -rf "$TEST_ROOT/out"
-if [[ $(uname) = Darwin ]]; then
-    expectStderr 1 nix-store --restore "$TEST_ROOT/out" < unnormalized.nar | grepQuiet "path '.*/out/â' already exists"
-else
-    nix-store --restore "$TEST_ROOT/out" < unnormalized.nar
+set +e
+unicodeTestOut=$(nix-store --restore "$TEST_ROOT/out" < unnormalized.nar 2>&1)
+unicodeTestCode=$?
+set -e
+
+touch "$TEST_ROOT/unicode-â" # non-canonical version
+touch "$TEST_ROOT/unicode-â"
+
+touchFilesCount=$(find "$TEST_ROOT" -maxdepth 1 -name "unicode-*" -type f | wc -l)
+
+if (( unicodeTestCode == 1 )); then
+    # If the command failed (MacOS or ZFS + normalization), checks that it failed
+    # with the expected "already exists" error, and that this is the same
+    # behavior as `touch`
+    echo "$unicodeTestOut" | grepQuiet "path '.*/out/â' already exists"
+
+    (( touchFilesCount == 1 ))
+elif (( unicodeTestCode == 0 )); then
+    # If the command succeeded, check that both files are present, and that this
+    # is the same behavior as `touch`
     [[ -e $TEST_ROOT/out/â ]]
     [[ -e $TEST_ROOT/out/â ]]
+
+    (( touchFilesCount == 2 ))
+else
+    # if the return code is neither 0 or 1, fail the test.
+    echo "NAR deserialization of files with the same Unicode normalization failed with unexpected return code $unicodeTestCode" >&2
+    exit 1
 fi
 
+rm -f "$TEST_ROOT/unicode-*"
+
 # Unpacking a NAR with a NUL character in a file name should fail.
 rm -rf "$TEST_ROOT/out"
 expectStderr 1 nix-store --restore "$TEST_ROOT/out" < nul.nar | grepQuiet "NAR contains invalid file name 'f"

tests/functional/package.nix

@@ -48,7 +48,7 @@ mkMesonDerivation (finalAttrs: {
   ];
 
   # Hack for sake of the dev shell
-  passthru.baseNativeBuildInputs = [
+  passthru.externalNativeBuildInputs = [
     meson
     ninja
     pkg-config
@@ -66,7 +66,7 @@ mkMesonDerivation (finalAttrs: {
     util-linux
   ];
 
-  nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [
+  nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [
     nix-cli
   ];
 
@@ -75,7 +75,6 @@ mkMesonDerivation (finalAttrs: {
     nix-expr
   ];
 
-
   preConfigure =
     # "Inline" .version so it's not a symlink, and includes the suffix.
     # Do the meson utils, without modification.

tests/functional/supplementary-groups.sh

@@ -9,7 +9,7 @@ needLocalStore "The test uses --store always so we would just be bypassing the d
 
 TODO_NixOS
 
-unshare --mount --map-root-user bash <<EOF
+unshare --mount --map-root-user -- bash -e -x <<EOF
   source common.sh
 
   # Avoid store dir being inside sandbox build-dir
@@ -24,15 +24,13 @@ unshare --mount --map-root-user bash <<EOF
   cmd=(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --no-out-link)
 
   # Fails with default setting
-  # TODO better error
   setLocalStore store1
-  expectStderr 1 "\${cmd[@]}" | grepQuiet "unable to start build process"
+  expectStderr 1 "\${cmd[@]}" | grepQuiet "setgroups failed"
 
   # Fails with `require-drop-supplementary-groups`
-  # TODO better error
   setLocalStore store2
   NIX_CONFIG='require-drop-supplementary-groups = true' \
-    expectStderr 1 "\${cmd[@]}" | grepQuiet "unable to start build process"
+    expectStderr 1 "\${cmd[@]}" | grepQuiet "setgroups failed"
 
   # Works without `require-drop-supplementary-groups`
   setLocalStore store3

tests/functional/tarball.sh

@@ -97,3 +97,17 @@ chmod +x "$TEST_ROOT/tar_root/foo"
 tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" .
 path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)"
 [[ $(cat "$path/foo") = bar ]]
+
+# Test a tarball with non-contiguous directory entries.
+rm -rf "$TEST_ROOT/tar_root"
+mkdir -p "$TEST_ROOT/tar_root/a/b"
+echo foo > "$TEST_ROOT/tar_root/a/b/foo"
+echo bla > "$TEST_ROOT/tar_root/bla"
+tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" .
+echo abc > "$TEST_ROOT/tar_root/bla"
+echo xyzzy > "$TEST_ROOT/tar_root/a/b/xyzzy"
+tar rvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" ./a/b/xyzzy ./bla
+path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)"
+[[ $(cat "$path/a/b/xyzzy") = xyzzy ]]
+[[ $(cat "$path/a/b/foo") = foo ]]
+[[ $(cat "$path/bla") = abc ]]

tests/nixos/fetchurl.nix

@@ -1,7 +1,7 @@
 # Test whether builtin:fetchurl properly performs TLS certificate
 # checks on HTTPS servers.
 
-{ lib, config, pkgs, ... }:
+{ pkgs, ... }:
 
 let
 
@@ -25,7 +25,7 @@ in
   name = "nss-preload";
 
   nodes = {
-    machine = { lib, pkgs, ... }: {
+    machine = { pkgs, ... }: {
       services.nginx = {
         enable = true;
 
@@ -60,13 +60,16 @@ in
     };
   };
 
-  testScript = { nodes, ... }: ''
+  testScript = ''
     machine.wait_for_unit("nginx")
     machine.wait_for_open_port(443)
 
     out = machine.succeed("curl https://good/index.html")
     assert out == "hello world\n"
 
+    out = machine.succeed("cat ${badCert}/cert.pem > /tmp/cafile.pem; curl --cacert /tmp/cafile.pem https://bad/index.html")
+    assert out == "foobar\n"
+
     # Fetching from a server with a trusted cert should work.
     machine.succeed("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://good/index.html\"; hash = \"sha256-qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A+4XSmaGSpEc=\"; }'")
 
@@ -74,5 +77,8 @@ in
     err = machine.fail("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }' 2>&1")
     print(err)
     assert "SSL certificate problem: self-signed certificate" in err
+
+    # Fetching from a server with a trusted cert should work via environment variable override.
+    machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'")
   '';
 }

tests/nixos/s3-binary-cache-store.nix

@@ -51,6 +51,9 @@ in {
 
     server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}")
 
+    # Test fetchurl on s3:// URLs while we're at it.
+    client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000&region=eu-west-1\"; }'")
+
     # Copy a package from the binary cache.
     client.fail("nix path-info ${pkgA}")
 

tests/unit/libexpr-support/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-store-test-support
 , nix-expr
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-util-test-support";
   inherit version;
 
@@ -37,14 +32,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-store-test-support
     nix-expr
@@ -66,10 +53,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

tests/unit/libexpr/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-expr
 , nix-expr-c
@@ -26,7 +21,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-expr-tests";
   inherit version;
 
@@ -42,12 +37,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-expr
     nix-expr-c
@@ -71,10 +60,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = runCommand "${finalAttrs.pname}-run" {

tests/unit/libfetchers/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-fetchers
 , nix-store-test-support
@@ -25,7 +20,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-fetchers-tests";
   inherit version;
 
@@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-fetchers
     nix-store-test-support
@@ -69,10 +58,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = runCommand "${finalAttrs.pname}-run" {

tests/unit/libflake/package.nix

@@ -1,12 +1,7 @@
 { lib
 , buildPackages
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonExecutable
 
 , nix-flake
 , nix-expr-test-support
@@ -25,7 +20,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonExecutable (finalAttrs: {
   pname = "nix-flake-tests";
   inherit version;
 
@@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   buildInputs = [
     nix-flake
     nix-expr-test-support
@@ -69,10 +58,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   passthru = {
     tests = {
       run = runCommand "${finalAttrs.pname}-run" {

tests/unit/libstore-support/package.nix

@@ -1,11 +1,6 @@
 { lib
 , stdenv
-, mkMesonDerivation
-, releaseTools
-
-, meson
-, ninja
-, pkg-config
+, mkMesonLibrary
 
 , nix-util-test-support
 , nix-store
@@ -21,7 +16,7 @@ let
   inherit (lib) fileset;
 in
 
-mkMesonDerivation (finalAttrs: {
+mkMesonLibrary (finalAttrs: {
   pname = "nix-store-test-support";
   inherit version;
 
@@ -37,14 +32,6 @@ mkMesonDerivation (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
   ];
 
-  outputs = [ "out" "dev" ];
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ];
-
   propagatedBuildInputs = [
     nix-util-test-support
     nix-store
@@ -66,10 +53,6 @@ mkMesonDerivation (finalAttrs: {
     LDFLAGS = "-fuse-ld=gold";
   };
 
-  separateDebugInfo = !stdenv.hostPlatform.isStatic;
-
-  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

tests/unit/libstore/derivation-advanced-attrs.cc

@@ -1,4 +1,3 @@
-#include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 #include <optional>
 
@@ -9,6 +8,7 @@
 #include "tests/characterization.hh"
 #include "parsed-derivations.hh"
 #include "types.hh"
+#include "json-utils.hh"
 
 namespace nix {