Merge 8bd591db03 into f51974d698

Fix `Interrupted` falling out of thread crash

.github/ISSUE_TEMPLATE/missing_documentation.md

@@ -23,7 +23,7 @@ assignees: ''
 - [ ] checked [open documentation issues and pull requests] for possible duplicates
 
 [latest Nix manual]: https://nixos.org/manual/nix/unstable/
-[source]: https://github.com/NixOS/nix/tree/master/doc/manual/src
+[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source
 [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation
 
 ## Priorities

.github/labeler.yml

@@ -9,7 +9,7 @@
     - any-glob-to-any-file: "CONTRIBUTING.md"
     - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*"
     - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md"
-    - any-glob-to-any-file: "doc/manual/src/contributing/**"
+    - any-glob-to-any-file: "doc/manual/source/contributing/**"
 
 "documentation":
   - changed-files:

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@V28
+    - uses: cachix/install-nix-action@v30
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -89,7 +89,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@V28
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - uses: cachix/cachix-action@v15
@@ -112,7 +112,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@V28
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -142,7 +142,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@V28
+    - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV

.gitignore

@@ -23,17 +23,17 @@ perl/Makefile.config
 /doc/manual/conf-file.json
 /doc/manual/language.json
 /doc/manual/xp-features.json
-/doc/manual/src/SUMMARY.md
-/doc/manual/src/SUMMARY-rl-next.md
-/doc/manual/src/store/types/*
-!/doc/manual/src/store/types/index.md.in
-/doc/manual/src/command-ref/new-cli
-/doc/manual/src/command-ref/conf-file.md
-/doc/manual/src/command-ref/experimental-features-shortlist.md
-/doc/manual/src/contributing/experimental-feature-descriptions.md
-/doc/manual/src/language/builtins.md
-/doc/manual/src/language/builtin-constants.md
-/doc/manual/src/release-notes/rl-next.md
+/doc/manual/source/SUMMARY.md
+/doc/manual/source/SUMMARY-rl-next.md
+/doc/manual/source/store/types/*
+!/doc/manual/source/store/types/index.md.in
+/doc/manual/source/command-ref/new-cli
+/doc/manual/source/command-ref/conf-file.md
+/doc/manual/source/command-ref/experimental-features-shortlist.md
+/doc/manual/source/contributing/experimental-feature-descriptions.md
+/doc/manual/source/language/builtins.md
+/doc/manual/source/language/builtin-constants.md
+/doc/manual/source/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh

CONTRIBUTING.md

@@ -79,7 +79,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
      - Functional tests – [`tests/functional/**.sh`](./tests/functional)
      - Unit tests – [`src/*/tests`](./src/)
      - Integration tests – [`tests/nixos/*`](./tests/nixos)
-   - [ ] User documentation in the [manual](./doc/manual/src)
+   - [ ] User documentation in the [manual](./doc/manual/source)
    - [ ] API documentation in header files
    - [ ] Code and comments are self-explanatory
    - [ ] Commit message explains **why** the change was made
@@ -90,7 +90,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
 ## Making changes to the Nix manual
 
 The Nix reference manual is hosted on https://nixos.org/manual/nix.
-The underlying source files are located in [`doc/manual/src`](./doc/manual/src).
+The underlying source files are located in [`doc/manual/source`](./doc/manual/source).
 For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files)
 For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html).
 

HACKING.md

@@ -1 +1 @@
-doc/manual/src/development/building.md
+doc/manual/source/development/building.md

doc/manual/.version

@@ -0,0 +1 @@
+../../.version

doc/manual/book.toml

@@ -1,5 +1,6 @@
 [book]
 title = "Nix Reference Manual"
+src = "source"
 
 [output.html]
 additional-css = ["custom.css"]
@@ -7,9 +8,21 @@ additional-js = ["redirects.js"]
 edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
 git-repository-url = "https://github.com/NixOS/nix"
 
+# Handles replacing @docroot@ with a path to ./source relative to that markdown file,
+# {{#include handlebars}}, and the @generated@ syntax used within these. it mostly
+# but not entirely replaces the links preprocessor (which we cannot simply use due
+# to @generated@ files living in a different directory to make meson happy). we do
+# not want to disable the links preprocessor entirely though because that requires
+# disabling *all* built-in preprocessors and selectively reenabling those we want.
+[preprocessor.substitute]
+command = "python3 ./substitute.py"
+before = ["anchors", "links"]
+
 [preprocessor.anchors]
 renderers = ["html"]
-command = "jq --from-file doc/manual/anchors.jq"
+command = "jq --from-file ./anchors.jq"
+
+[output.markdown]
 
 [output.linkcheck]
 # no Internet during the build (in the sandbox)

doc/manual/generate-deps.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import glob
+import sys
+
+# meson expects makefile-style dependency declarations, i.e.
+#
+#   target: dependency...
+#
+# meson seems to pass depfiles straight on to ninja even though
+# it also parses the file itself (or at least has code to do so
+# in its tree), so we must live by ninja's rules: only slashes,
+# spaces and octothorpes can be escaped, anything else is taken
+# literally. since the rules for these aren't even the same for
+# all three we will just fail when we encounter any of them (if
+# asserts are off for some reason the depfile will likely point
+# to nonexistant paths, making everything phony and thus fine.)
+for path in glob.glob(sys.argv[1] + '/**', recursive=True):
+    assert '\\' not in path
+    assert ' ' not in path
+    assert '#' not in path
+    print("ignored:", path)

doc/manual/generate-store-types.nix

@@ -21,7 +21,7 @@ let
 
   "index.md" = replaceStrings
     [ "@store-types@" ] [ index ]
-    (readFile ./src/store/types/index.md.in);
+    (readFile ./source/store/types/index.md.in);
 
   tableOfContents =
     let

doc/manual/local.mk

@@ -4,8 +4,8 @@
 doc_nix = $(nix_PATH)
 
 MANUAL_SRCS := \
-	$(call rwildcard, $(d)/src, *.md) \
-	$(call rwildcard, $(d)/src, */*.md)
+	$(call rwildcard, $(d)/source, *.md) \
+	$(call rwildcard, $(d)/source, */*.md)
 
 man-pages := $(foreach n, \
 	nix-env.1 nix-store.1 \
@@ -18,11 +18,11 @@ man-pages := $(foreach n, \
 , $(d)/$(n))
 
 # man pages for subcommands
-# convert from `$(d)/src/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1`
+# convert from `$(d)/source/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1`
 # FIXME: unify with how nix3-cli man pages are generated
 man-pages += $(foreach subcommand, \
-	$(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/src/command-ref/nix-*/*.md)), \
-	$(d)/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand)))))
+	$(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/source/command-ref/nix-*/*.md)), \
+	$(d)/$(subst /,-,$(subst $(d)/source/command-ref/,,$(subst .md,.1,$(subcommand)))))
 
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
@@ -49,11 +49,11 @@ define process-includes
 	done < <(grep '{{#include' $(1))
 endef
 
-$(d)/nix-env-%.1: $(d)/src/command-ref/nix-env/%.md
+$(d)/nix-env-%.1: $(d)/source/command-ref/nix-env/%.md
 	@printf "Title: %s\n\n" "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" > $^.tmp
 	$(render-subcommand)
 
-$(d)/nix-store-%.1: $(d)/src/command-ref/nix-store/%.md
+$(d)/nix-store-%.1: $(d)/source/command-ref/nix-store/%.md
 	@printf -- 'Title: %s\n\n' "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" > $^.tmp
 	$(render-subcommand)
 
@@ -69,50 +69,50 @@ define render-subcommand
 endef
 
 
-$(d)/%.1: $(d)/src/command-ref/%.md
+$(d)/%.1: $(d)/source/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp
 	@cat $^ >> $^.tmp
 	@$(call process-includes,$^,$^.tmp)
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/%.8: $(d)/src/command-ref/%.md
+$(d)/%.8: $(d)/source/command-ref/%.md
 	@printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp
 	@cat $^ >> $^.tmp
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=8 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
+$(d)/nix.conf.5: $(d)/source/command-ref/conf-file.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
 	@$(call process-includes,$^,$^.tmp)
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
+$(d)/nix-profiles.5: $(d)/source/command-ref/files/profiles.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md
+$(d)/source/SUMMARY.md: $(d)/source/SUMMARY.md.in $(d)/source/SUMMARY-rl-next.md $(d)/source/store/types $(d)/source/command-ref/new-cli $(d)/source/development/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix)
+$(d)/source/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/source/store/types/index.md.in $(doc_nix)
 	@# FIXME: build out of tree!
 	@rm -rf $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores'
 	@# do not destroy existing contents
 	@mv $@.tmp/* $@/
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
+$(d)/source/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix)
-	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
+$(d)/source/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/source/command-ref/conf-file-prefix.md $(d)/source/command-ref/experimental-features-shortlist.md $(doc_nix)
+	@cat doc/manual/source/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
@@ -124,12 +124,12 @@ $(d)/conf-file.json: $(doc_nix)
 	$(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
+$(d)/source/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
+$(d)/source/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
@@ -138,10 +138,10 @@ $(d)/xp-features.json: $(doc_nix)
 	$(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
-	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
+$(d)/source/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/source/language/builtins-prefix.md $(doc_nix)
+	@cat doc/manual/source/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
-	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
+	@cat doc/manual/source/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/language.json: $(doc_nix)
@@ -149,7 +149,7 @@ $(d)/language.json: $(doc_nix)
 	@mv $@.tmp $@
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)
-$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
+$(d)/source/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
 	@if type -p changelog-d > /dev/null; then \
 		echo "  GEN   " $@; \
 		changelog-d doc/manual/rl-next > $@; \
@@ -158,7 +158,7 @@ $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
 		true > $@; \
 	fi
 
-$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+$(d)/source/SUMMARY-rl-next.md: $(d)/source/release-notes/rl-next.md
 	$(trace-gen) true
 	@if [ -s $< ]; then \
 		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
@@ -194,9 +194,9 @@ $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages
 	@mkdir -p $(DESTDIR)$$(dirname $@)
 	$(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@)
 
-doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
+doc/manual/generated/man1/nix3-manpages: $(d)/source/command-ref/new-cli
 	@mkdir -p $(DESTDIR)$$(dirname $@)
-	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
+	$(trace-gen) for i in doc/manual/source/command-ref/new-cli/*.md; do \
 		name=$$(basename $$i .md); \
 		tmpFile=$$(mktemp); \
 		if [[ $$name = SUMMARY ]]; then continue; fi; \
@@ -211,7 +211,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/source/SUMMARY.md $(d)/source/store/types $(d)/source/command-ref/new-cli $(d)/source/development/experimental-feature-descriptions.md $(d)/source/command-ref/conf-file.md $(d)/source/language/builtins.md $(d)/source/release-notes/rl-next.md $(d)/source/figures $(d)/source/favicon.png $(d)/source/favicon.svg
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \
@@ -219,12 +219,17 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/
 			$(call process-includes,$$file,$$file); \
 		done; \
 		find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \
-			docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \
+			docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/source)"; \
 			sed -i "s,@docroot@,$$docroot,g" "$$file"; \
 		done; \
 		set -euo pipefail; \
-		RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
-			| { grep -Fv "because fragment resolution isn't implemented" || :; }; \
+		( \
+		    cd "$$tmp/manual"; \
+		    RUST_LOG=warn \
+		        MDBOOK_SUBSTITUTE_SEARCH=$(d)/source \
+		        mdbook build -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \
+			    | { grep -Fv "because fragment resolution isn't implemented" || :; } \
+		); \
 		rm -rf "$$tmp/manual"
 	@rm -rf $(DESTDIR)$(docdir)/manual
 	@mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual

doc/manual/meson.build

@@ -0,0 +1,353 @@
+project('nix-manual',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+nix = find_program('nix', native : true)
+
+mdbook = find_program('mdbook', native : true)
+bash = find_program('bash', native : true)
+
+pymod = import('python')
+python = pymod.find_installation('python3')
+
+nix_env_for_docs = {
+  'HOME': '/dummy',
+  'NIX_CONF_DIR': '/dummy',
+  'NIX_SSL_CERT_FILE': '/dummy/no-ca-bundle.crt',
+  'NIX_STATE_DIR': '/dummy',
+  'NIX_CONFIG': 'cores = 0',
+}
+
+nix_for_docs = [nix, '--experimental-features', 'nix-command']
+nix_eval_for_docs_common = nix_for_docs + [
+  'eval',
+  '-I', 'nix=' + meson.current_source_dir(),
+  '--store', 'dummy://',
+  '--impure',
+]
+nix_eval_for_docs = nix_eval_for_docs_common + '--raw'
+
+conf_file_json = custom_target(
+  command : nix_for_docs + ['config', 'show', '--json'],
+  capture : true,
+  output : 'conf-file.json',
+  env : nix_env_for_docs,
+)
+
+language_json = custom_target(
+  command: [nix, '__dump-language'],
+  output : 'language.json',
+  capture : true,
+  env : nix_env_for_docs,
+)
+
+nix3_cli_json = custom_target(
+  command : [nix, '__dump-cli'],
+  capture : true,
+  output : 'nix.json',
+  env : nix_env_for_docs,
+)
+
+generate_manual_deps = files(
+  'generate-deps.py',
+)
+
+# Generates types
+subdir('source/store')
+# Generates builtins.md and builtin-constants.md.
+subdir('source/language')
+# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md.
+subdir('source/command-ref')
+# Generates experimental-feature-descriptions.md.
+subdir('source/development')
+# Generates rl-next-generated.md.
+subdir('source/release-notes')
+subdir('source')
+
+# Hacky way to figure out if `nix` is an `ExternalProgram` or
+# `Exectuable`. Only the latter can occur in custom target input lists.
+if nix.full_path().startswith(meson.build_root())
+  nix_input = nix
+else
+  nix_input = []
+endif
+
+manual = custom_target(
+  'manual',
+  command : [
+    bash,
+    '-euo', 'pipefail',
+    '-c',
+    '''
+        @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
+        @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
+        rsync -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
+        (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
+        rm -rf @2@/manual
+        mv @2@/html @2@/manual
+        find @2@/manual -iname meson.build -delete
+    '''.format(
+      python.full_path(),
+      mdbook.full_path(),
+      meson.current_build_dir(),
+    ),
+  ],
+  input : [
+    generate_manual_deps,
+    'substitute.py',
+    'book.toml',
+    'anchors.jq',
+    'custom.css',
+    nix3_cli_files,
+    experimental_features_shortlist_md,
+    experimental_feature_descriptions_md,
+    types_dir,
+    conf_file_md,
+    builtins_md,
+    rl_next_generated,
+    summary_rl_next,
+    nix_input,
+  ],
+  output : [
+    'manual',
+    'markdown',
+  ],
+  depfile : 'manual.d',
+  env : {
+    'RUST_LOG': 'info',
+    'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'source',
+  },
+)
+manual_html = manual[0]
+manual_md = manual[1]
+
+install_subdir(
+  manual_html.full_path(),
+  install_dir : get_option('datadir') / 'doc/nix',
+)
+
+nix_nested_manpages = [
+  [ 'nix-env',
+    [
+      'delete-generations',
+      'install',
+      'list-generations',
+      'query',
+      'rollback',
+      'set-flag',
+      'set',
+      'switch-generation',
+      'switch-profile',
+      'uninstall',
+      'upgrade',
+    ],
+  ],
+  [ 'nix-store',
+    [
+      'add-fixed',
+      'add',
+      'delete',
+      'dump-db',
+      'dump',
+      'export',
+      'gc',
+      'generate-binary-cache-key',
+      'import',
+      'load-db',
+      'optimise',
+      'print-env',
+      'query',
+      'read-log',
+      'realise',
+      'repair-path',
+      'restore',
+      'serve',
+      'verify',
+      'verify-path',
+    ],
+  ],
+]
+
+foreach command : nix_nested_manpages
+  foreach page : command[1]
+    title = command[0] + ' --' + page
+    section = '1'
+    custom_target(
+      command : [
+        bash,
+        files('./render-manpage.sh'),
+        '--out-no-smarty',
+        title,
+        section,
+        '@INPUT0@/command-ref' / command[0] / (page + '.md'),
+        '@OUTPUT0@',
+      ],
+      input : [
+        manual_md,
+        nix_input,
+      ],
+      output : command[0] + '-' + page + '.1',
+      install : true,
+      install_dir : get_option('mandir') / 'man1',
+    )
+  endforeach
+endforeach
+
+nix3_manpages = [
+  'nix3-build',
+  'nix3-bundle',
+  'nix3-config',
+  'nix3-config-show',
+  'nix3-copy',
+  'nix3-daemon',
+  'nix3-derivation-add',
+  'nix3-derivation',
+  'nix3-derivation-show',
+  'nix3-develop',
+  #'nix3-doctor',
+  'nix3-edit',
+  'nix3-eval',
+  'nix3-flake-archive',
+  'nix3-flake-check',
+  'nix3-flake-clone',
+  'nix3-flake-info',
+  'nix3-flake-init',
+  'nix3-flake-lock',
+  'nix3-flake',
+  'nix3-flake-metadata',
+  'nix3-flake-new',
+  'nix3-flake-prefetch',
+  'nix3-flake-show',
+  'nix3-flake-update',
+  'nix3-fmt',
+  'nix3-hash-file',
+  'nix3-hash',
+  'nix3-hash-path',
+  'nix3-hash-to-base16',
+  'nix3-hash-to-base32',
+  'nix3-hash-to-base64',
+  'nix3-hash-to-sri',
+  'nix3-help',
+  'nix3-help-stores',
+  'nix3-key-convert-secret-to-public',
+  'nix3-key-generate-secret',
+  'nix3-key',
+  'nix3-log',
+  'nix3-nar-cat',
+  'nix3-nar-dump-path',
+  'nix3-nar-ls',
+  'nix3-nar',
+  'nix3-path-info',
+  'nix3-print-dev-env',
+  'nix3-profile-diff-closures',
+  'nix3-profile-history',
+  'nix3-profile-install',
+  'nix3-profile-list',
+  'nix3-profile',
+  'nix3-profile-remove',
+  'nix3-profile-rollback',
+  'nix3-profile-upgrade',
+  'nix3-profile-wipe-history',
+  'nix3-realisation-info',
+  'nix3-realisation',
+  'nix3-registry-add',
+  'nix3-registry-list',
+  'nix3-registry',
+  'nix3-registry-pin',
+  'nix3-registry-remove',
+  'nix3-repl',
+  'nix3-run',
+  'nix3-search',
+  #'nix3-shell',
+  'nix3-store-add-file',
+  'nix3-store-add-path',
+  'nix3-store-cat',
+  'nix3-store-copy-log',
+  'nix3-store-copy-sigs',
+  'nix3-store-delete',
+  'nix3-store-diff-closures',
+  'nix3-store-dump-path',
+  'nix3-store-gc',
+  'nix3-store-ls',
+  'nix3-store-make-content-addressed',
+  'nix3-store',
+  'nix3-store-optimise',
+  'nix3-store-path-from-hash-part',
+  'nix3-store-ping',
+  'nix3-store-prefetch-file',
+  'nix3-store-repair',
+  'nix3-store-sign',
+  'nix3-store-verify',
+  'nix3-upgrade-nix',
+  'nix3-why-depends',
+  'nix',
+]
+
+foreach page : nix3_manpages
+  section = '1'
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      page,
+      section,
+      '@INPUT1@/command-ref/new-cli/@0@.md'.format(page),
+      '@OUTPUT@',
+    ],
+    input : [
+      files('./render-manpage.sh'),
+      manual_md,
+      nix_input,
+    ],
+    output : page + '.1',
+    install : true,
+    install_dir : get_option('mandir') / 'man1',
+  )
+endforeach
+
+nix_manpages = [
+  [ 'nix-env', 1 ],
+  [ 'nix-store', 1 ],
+  [ 'nix-build', 1 ],
+  [ 'nix-shell', 1 ],
+  [ 'nix-instantiate', 1 ],
+  [ 'nix-collect-garbage', 1 ],
+  [ 'nix-prefetch-url', 1 ],
+  [ 'nix-channel', 1 ],
+  [ 'nix-hash', 1 ],
+  [ 'nix-copy-closure', 1 ],
+  [ 'nix.conf', 5, conf_file_md.full_path() ],
+  [ 'nix-daemon', 8 ],
+  [ 'nix-profiles', 5, 'files/profiles.md' ],
+]
+
+foreach entry : nix_manpages
+  title = entry[0]
+  # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md,
+  # rather than a stem identical to its mdbook source.
+  # Therefore we use an optional third element of this array to override the name pattern
+  md_file = entry.get(2, title + '.md')
+  section = entry[1].to_string()
+  md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file)
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      title,
+      section,
+      md_file_resolved,
+      '@OUTPUT@',
+    ],
+    input : [
+      files('./render-manpage.sh'),
+      manual_md,
+      entry.get(3, []),
+      nix_input,
+    ],
+    output : '@0@.@1@'.format(entry[0], entry[1]),
+    install : true,
+    install_dir : get_option('mandir') / 'man@0@'.format(entry[1]),
+  )
+endforeach

doc/manual/package.nix

@@ -0,0 +1,71 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, lowdown
+, mdbook
+, mdbook-linkcheck
+, jq
+, python3
+, rsync
+, nix-cli
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-manual";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.difference
+    (fileset.unions [
+      ../../.version
+      # Too many different types of files to filter for now
+      ../../doc/manual
+      ./.
+    ])
+    # Do a blacklist instead
+    ../../doc/manual/package.nix;
+
+  # TODO the man pages should probably be separate
+  outputs = [ "out" "man" ];
+
+  # Hack for sake of the dev shell
+  passthru.externalNativeBuildInputs = [
+    meson
+    ninja
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+    jq
+    python3
+    rsync
+  ];
+
+  nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [
+    nix-cli
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''$out/nix-support
+    echo "doc manual ''$out/share/doc/nix/manual" >> ''$out/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

doc/manual/redirects.js

@@ -1,7 +1,7 @@
 // redirect rules for URL fragments (client-side) to prevent link rot.
 // this must be done on the client side, as web servers do not see the fragment part of the URL.
 // it will only work with JavaScript enabled in the browser, but this is the best we can do here.
-// see src/_redirects for path redirects (server-side)
+// see source/_redirects for path redirects (server-side)
 
 // redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.

doc/manual/remove_before_wrapper.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import shutil
+import typing as t
+
+def main():
+    if len(sys.argv) < 4 or '--' not in sys.argv:
+        print("Usage: remove-before-wrapper <output> -- <nix command...>")
+        sys.exit(1)
+
+    # Extract the parts
+    output: str = sys.argv[1]
+    nix_command_idx: int = sys.argv.index('--') + 1
+    nix_command: t.List[str] = sys.argv[nix_command_idx:]
+
+    output_temp: str = output + '.tmp'
+
+    # Remove the output and temp output in case they exist
+    shutil.rmtree(output, ignore_errors=True)
+    shutil.rmtree(output_temp, ignore_errors=True)
+
+    # Execute nix command with `--write-to` tempary output
+    nix_command_write_to = nix_command + ['--write-to', output_temp]
+    subprocess.run(nix_command_write_to, check=True)
+
+    # Move the temporary output to the intended location
+    os.rename(output_temp, output)
+
+if __name__ == "__main__":
+    main()

doc/manual/render-manpage.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+lowdown_args=
+
+if [ "$1" = --out-no-smarty ]; then
+    lowdown_args=--out-no-smarty
+    shift
+fi
+
+[ "$#" = 4 ] || {
+    echo "wrong number of args passed" >&2
+    exit 1
+}
+
+title="$1"
+section="$2"
+infile="$3"
+outfile="$4"
+
+(
+    printf "Title: %s\n\n" "$title"
+    cat "$infile"
+) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile"

doc/manual/rl-next/build-hook-default.md

@@ -16,7 +16,7 @@ This has a small adverse affect on remote building --- the `build-remote` execut
 This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work.
 
 Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221).
-There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder.
+There should simply be no need to have an extra, intermediate layer of remote-procedure-calling when we want to connect to a remote builder.
 The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/index.md) i.e. `Store` subclass. <!-- we normally don't mention classes, but consider that this release note is about a library use case -->
 
-The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above.
+The Perl bindings no longer expose `getBinDir` either, since the underlying C++ libraries those bindings wrap no longer know the location of installed binaries as described above.

doc/manual/rl-next/verify-tls.md

@@ -0,0 +1,8 @@
+---
+synopsis: "`<nix/fetchurl.nix>` uses TLS verification"
+prs: [11585]
+---
+
+Previously `<nix/fetchurl.nix>` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `<nix/fetchurl.nix>` will now fail if you're fetching from a HTTPS server that does not have a valid certificate.
+
+`<nix/fetchurl.nix>` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue.

doc/manual/source/advanced-topics/distributed-builds.md

@@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can
 test whether connecting to the remote Nix instance works, e.g.
 
 ```console
-$ nix store ping --store ssh://mac
+$ nix store info --store ssh://mac
 ```
 
 will try to connect to the machine named `mac`. It is possible to
 specify an SSH identity file as part of the remote store URI, e.g.
 
 ```console
-$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key
+$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key
 ```
 
 Since builds should be non-interactive, the key should not have a

doc/manual/source/command-ref/meson.build

@@ -0,0 +1,63 @@
+xp_features_json = custom_target(
+  command : [nix, '__dump-xp-features'],
+  capture : true,
+  output : 'xp-features.json',
+)
+
+experimental_features_shortlist_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile ./@INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features-shortlist.nix',
+    xp_features_json,
+  ],
+  output : 'experimental-features-shortlist.md',
+  capture : true,
+  env : nix_env_for_docs,
+)
+
+nix3_cli_files = custom_target(
+  command : [
+    python.full_path(),
+    '@INPUT0@',
+    '@OUTPUT@',
+    '--'
+  ] + nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT1@ true (builtins.readFile ./@INPUT2@)',
+  ],
+  input : [
+    '../../remove_before_wrapper.py',
+    '../../generate-manpage.nix',
+    nix3_cli_json,
+  ],
+  output : 'new-cli',
+  env : nix_env_for_docs,
+)
+
+conf_file_md_body = custom_target(
+  command : [
+    nix_eval_for_docs,
+    '--expr',
+    'import @INPUT0@ { prefix = "conf"; } (builtins.fromJSON (builtins.readFile ./@INPUT1@))',
+  ],
+  capture : true,
+  input : [
+    '../../generate-settings.nix',
+    conf_file_json,
+  ],
+  output : 'conf-file.body.md',
+  env : nix_env_for_docs,
+)
+
+conf_file_md = custom_target(
+  command : [ 'cat', '@INPUT0@', '@INPUT1@' ],
+  capture : true,
+  input : [
+    'conf-file-prefix.md',
+    conf_file_md_body,
+  ],
+  output : 'conf-file.md',
+)

doc/manual/source/command-ref/nix-env.md

@@ -62,7 +62,7 @@ These pages can be viewed offline:
 
 Several operations, such as [`nix-env --query`](./nix-env/query.md) and [`nix-env --install`](./nix-env/install.md), take a list of *arguments* that specify the packages on which to operate.
 
-Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-names):
+Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-name):
 
 - `name`: Everything up to but not including the first dash (`-`) that is *not* followed by a letter.
 - `version`: The rest, excluding the separating dash.

doc/manual/source/command-ref/nix-store/query.md

@@ -104,7 +104,7 @@ symlink.
 
   Prints a set of derivation files (`.drv`) which are supposed produce
   said paths when realized. Might print nothing, for example for source paths
-  or paths subsituted from a binary cache.
+  or paths substituted from a binary cache.
 
 - `--graph`
 
@@ -241,4 +241,3 @@ $ nix-store --query --roots $(which svn)
 /nix/var/nix/profiles/default-82-link
 /home/eelco/.local/state/nix/profiles/profile-97-link
 ```
-

doc/manual/source/command-ref/opt-common.md

@@ -1,3 +1,7 @@
+<!-- Some of the options documented here are hardcopied from
+     src/libcmd/common-eval-args.cc
+-->
+
 # Common Options
 
 Most Nix commands accept the following command-line options:
@@ -161,6 +165,14 @@ Most Nix commands accept the following command-line options:
   You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`.
   (Note that since the argument is a Nix string literal, you have to escape the quotes.)
 
+- <span id="opt-arg-from-file">[`--arg-from-file`](#opt-arg-from-file)</span> *name* *path*
+
+  Pass the contents of file *path* as the argument *name* to Nix functions.
+
+- <span id="opt-arg-from-stdin">[`--arg-from-stdin`](#opt-arg-from-stdin)</span> *name*
+
+  Pass the contents of stdin as the argument *name* to Nix functions.
+
 - <span id="opt-argstr">[`--argstr`](#opt-argstr)</span> *name* *value*
 
   This option is like `--arg`, only the value is not a Nix expression but a string.
@@ -179,6 +191,10 @@ Most Nix commands accept the following command-line options:
   attribute of the fourth element of the array in the `foo` attribute
   of the top-level expression.
 
+- <span id="opt-eval-store">[`--eval-store`](#opt-eval-store)</span> *store-url*
+
+  The [URL to the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. where to store derivations (`.drv` files) and inputs referenced by them.
+
 - <span id="opt-expr">[`--expr`](#opt-expr)</span> / `-E`
 
   Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions.
@@ -194,6 +210,10 @@ Most Nix commands accept the following command-line options:
 
   Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH).
 
+- <span id="opt-impure">[`--impure`](#opt-impure)</span>
+
+  Allow access to mutable paths and repositories.
+
 - <span id="opt-option">[`--option`](#opt-option)</span> *name* *value*
 
   Set the Nix configuration option *name* to *value*.

doc/manual/source/development/documentation.md

@@ -35,7 +35,7 @@ In order to reflect changes to the [Makefile for the manual], clear all generate
 [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk
 
 ```console
-rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES
+rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/source/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES
 ```
 
 ## Style guide
@@ -182,7 +182,7 @@ Please observe these guidelines to ease reviews:
 
 `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own.
 
-If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory.
+If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/source` directory.
 
 If the `@docroot@` literal appears in an error message from the [`mdbook-linkcheck`] tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it.
 See existing `@docroot@` logic in the [Makefile for the manual].

doc/manual/source/development/meson.build

@@ -0,0 +1,12 @@
+experimental_feature_descriptions_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features.nix',
+    xp_features_json,
+  ],
+  capture : true,
+  output : 'experimental-feature-descriptions.md',
+)

doc/manual/source/development/testing.md

@@ -162,7 +162,7 @@ ninja: no work to do.
 Individual tests can be run with `meson`:
 
 ```shell-session
-$ meson test ${testName}
+$ meson test --verbose ${testName}
 ninja: Entering directory `/home/jcericson/src/nix/master/build'
 ninja: no work to do.
 1/1 nix-functional-tests:main / ${testName}        OK               0.41s
@@ -177,7 +177,11 @@ Timeout:            0
 Full log written to /home/jcericson/src/nix/master/build/meson-logs/testlog.txt
 ```
 
-or without `meson`, showing the output:
+The `--verbose` flag will make Meson also show the console output of each test for easier debugging.
+The test script will then be traced with `set -x` and the output displayed as it happens,
+regardless of whether the test succeeds or fails.
+
+Tests can be also run directly without `meson`:
 
 ```shell-session
 $ TEST_NAME=${testName} NIX_REMOTE='' PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) tests/functional/${testName}.sh
@@ -188,8 +192,6 @@ output from bar
 ...
 ```
 
-The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
-
 ### Debugging failing functional tests
 
 When a functional test fails, it usually does so somewhere in the middle of the script.