From eb03773d0bb93b99ac6389d526a0beb064b79bb6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jul 2024 15:05:22 -0400 Subject: [PATCH] Meson the manual Co-Authored-By: Qyriad Co-Authored-By: eldritch horrors --- doc/manual/.version | 1 + doc/manual/generate-deps.py | 22 ++ doc/manual/json-to-tree.py | 61 ++++ doc/manual/meson.build | 371 +++++++++++++++++++++++ doc/manual/package.nix | 61 ++++ doc/manual/render-manpage.sh | 25 ++ doc/manual/src/command-ref/meson.build | 67 ++++ doc/manual/src/contributing/meson.build | 15 + doc/manual/src/language/meson.build | 13 + doc/manual/src/release-notes/meson.build | 22 ++ doc/manual/src/store/meson.build | 13 + flake.nix | 2 + meson.build | 3 + packaging/components.nix | 1 + packaging/everything.nix | 2 + packaging/hydra.nix | 3 + src/nix-manual | 1 + 17 files changed, 683 insertions(+) create mode 120000 doc/manual/.version create mode 100755 doc/manual/generate-deps.py create mode 100755 doc/manual/json-to-tree.py create mode 100644 doc/manual/meson.build create mode 100644 doc/manual/package.nix create mode 100755 doc/manual/render-manpage.sh create mode 100644 doc/manual/src/command-ref/meson.build create mode 100644 doc/manual/src/contributing/meson.build create mode 100644 doc/manual/src/language/meson.build create mode 100644 doc/manual/src/release-notes/meson.build create mode 100644 doc/manual/src/store/meson.build create mode 120000 src/nix-manual diff --git a/doc/manual/.version b/doc/manual/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/doc/manual/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/manual/generate-deps.py b/doc/manual/generate-deps.py new file mode 100755 index 000000000..297bd3939 --- /dev/null +++ b/doc/manual/generate-deps.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import glob +import sys + +# meson expects makefile-style dependency declarations, i.e. +# +# target: dependency... +# +# meson seems to pass depfiles straight on to ninja even though +# it also parses the file itself (or at least has code to do so +# in its tree), so we must live by ninja's rules: only slashes, +# spaces and octothorpes can be escaped, anything else is taken +# literally. since the rules for these aren't even the same for +# all three we will just fail when we encounter any of them (if +# asserts are off for some reason the depfile will likely point +# to nonexistant paths, making everything phony and thus fine.) +for path in glob.glob(sys.argv[1] + '/**', recursive=True): + assert '\\' not in path + assert ' ' not in path + assert '#' not in path + print("ignored:", path) diff --git a/doc/manual/json-to-tree.py b/doc/manual/json-to-tree.py new file mode 100755 index 000000000..27c8e417c --- /dev/null +++ b/doc/manual/json-to-tree.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +""" +This script is a helper for this project's Meson buildsystem, to replace its +usage of `nix eval --write-to`. Writing a JSON object as a nested directory +tree is more generic, easier to maintain, and far, far less cursed. Nix +has 'good' support for JSON output. Let's just use it. +""" + +import argparse +from pathlib import Path +import json +import sys + +name = 'json-to-tree.py' + +def log(*args, **kwargs): + kwargs['file'] = sys.stderr + return print(f'{name}:', *args, **kwargs) + +def write_dict_to_directory(current_directory: Path, data: dict, files_written=0): + current_directory.mkdir(parents=True, exist_ok=True) + for key, value in data.items(): + nested_path = current_directory / key + match value: + case dict(nested_data): + files_written += write_dict_to_directory(nested_path, nested_data) + + case str(content): + nested_path.write_text(content) + files_written += 1 + + case rest: + assert False, \ + f'should have been called on a dict or string, not {type(rest)=}\n\t{rest=}' + + return files_written + +def main(): + parser = argparse.ArgumentParser(name) + parser.add_argument('-i', '--input', type=argparse.FileType('r'), default='-', + help='The JSON input to operate on and output as a directory tree', + ) + parser.add_argument('-o', '--output', type=Path, required=True, + help='The place to put the directory tree', + ) + args = parser.parse_args() + + json_string = args.input.read() + + try: + data = json.loads(json_string) + except json.JSONDecodeError: + log(f'could not decode JSON from input: {json_string}') + raise + + + files_written = write_dict_to_directory(args.output, data) + log(f'wrote {files_written} files') + +sys.exit(main()) diff --git a/doc/manual/meson.build b/doc/manual/meson.build new file mode 100644 index 000000000..a9794991c --- /dev/null +++ b/doc/manual/meson.build @@ -0,0 +1,371 @@ +project('nix-manual', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +nix = find_program('nix', native : true) + +mdbook = find_program('mdbook', native : true) +bash = find_program('bash', native : true) + +pymod = import('python') +python = pymod.find_installation('python3') + +nix_env_for_docs = { + 'HOME': '/dummy', + 'NIX_CONF_DIR': '/dummy', + 'NIX_SSL_CERT_FILE': '/dummy/no-ca-bundle.crt', + 'NIX_STATE_DIR': '/dummy', + 'NIX_CONFIG': 'cores = 0', +} + +nix_for_docs = [ nix, '--experimental-features', 'nix-command' ] +nix_eval_for_docs_common = nix_for_docs + [ + 'eval', + '-I', 'nix=' + meson.current_source_dir(), + '--store', 'dummy://', + '--impure', +] +nix_eval_for_docs = nix_eval_for_docs_common + '--raw' + +conf_file_json = custom_target( + command : nix_for_docs + [ 'show-config', '--json' ], + capture : true, + output : 'conf-file.json', + env : nix_env_for_docs, +) + +nix_conf_file_md_body = custom_target( + command : nix_eval_for_docs + [ + '--expr', + 'import @INPUT1@ { prefix = "conf"; } (builtins.fromJSON (builtins.readFile ./@INPUT0@))', + ], + capture : true, + input : [ + conf_file_json, + 'utils.nix', + 'generate-settings.nix', + 'src' / 'command-ref' / 'conf-file-prefix.md', + 'src' / 'command-ref' / 'experimental-features-shortlist.md' + ], + output : 'conf-file.md.body', + env : nix_env_for_docs, +) + +nix_conf_file_md = custom_target( + command : [ 'cat', '@INPUT@' ], + capture : true, + input : [ + 'src/command-ref/conf-file-prefix.md', + nix_conf_file_md_body, + ], + output : 'conf-file.md', +) + +nix_exp_features_json = custom_target( + command : [ nix, '__dump-xp-features' ], + capture : true, + output : 'xp-features.json', +) + +language_json = custom_target( + command: [nix, '__dump-language'], + output : 'language.json', + capture : true, + env : nix_env_for_docs, +) + +nix3_cli_json = custom_target( + command : [ nix, '__dump-cli' ], + capture : true, + output : 'nix.json', + env : nix_env_for_docs, +) + +generate_manual_deps = files( + 'generate-deps.py', +) + +# Generates types +subdir('src/store') +# Generates builtins.md and builtin-constants.md. +subdir('src/language') +# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md. +subdir('src/command-ref') +# Generates experimental-feature-descriptions.md. +subdir('src/contributing') +# Generates rl-next-generated.md. +subdir('src/release-notes') + +manual = custom_target( + 'manual', + command : [ + bash, + '-euo', 'pipefail', + '-c', + ''' + @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ + cd @SOURCE_ROOT@ + @1@ build doc/manual -d @2@ | { grep -Fv "because fragment resolution isn't implemented" || :; } + rm -rf @2@/manual + mv @2@/html @2@/manual + find @2@/manual -iname meson.build -delete + '''.format( + python.full_path(), + mdbook.full_path(), + meson.current_build_dir(), + ), + ], + input : [ + generate_manual_deps, + 'book.toml', + 'anchors.jq', + 'custom.css', + nix3_cli_files, + experimental_features_shortlist_md, + experimental_feature_descriptions_md, + conf_file_md, + builtins_md, + rl_next_generated, + nix, + ], + output : [ + 'manual', + 'markdown', + ], + depfile : 'manual.d', + env : { + 'RUST_LOG': 'info', + 'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src', + }, +) +manual_html = manual[0] +manual_md = manual[1] + +install_subdir( + manual_html.full_path(), + install_dir : get_option('datadir') / 'doc/nix', +) + +nix_nested_manpages = [ + [ 'nix-env', + [ + 'delete-generations', + 'install', + 'list-generations', + 'query', + 'rollback', + 'set-flag', + 'set', + 'switch-generation', + 'switch-profile', + 'uninstall', + 'upgrade', + ], + ], + [ 'nix-store', + [ + 'add-fixed', + 'add', + 'delete', + 'dump-db', + 'dump', + 'export', + 'gc', + 'generate-binary-cache-key', + 'import', + 'load-db', + 'optimise', + 'print-env', + 'query', + 'read-log', + 'realise', + 'repair-path', + 'restore', + 'serve', + 'verify', + 'verify-path', + ], + ], +] + +foreach command : nix_nested_manpages + foreach page : command[1] + title = command[0] + ' --' + page + section = '1' + custom_target( + command : [ + './render-manpage.sh', + '--out-no-smarty', + title, + section, + '@INPUT0@/command-ref' / command[0] / (page + '.md'), + '@OUTPUT0@', + ], + input : [ + manual_md, + nix, + ], + output : command[0] + '-' + page + '.1', + install : true, + install_dir : get_option('mandir') / 'man1', + ) + endforeach +endforeach + +nix3_manpages = [ + 'nix3-build', + 'nix3-bundle', + 'nix3-config', + 'nix3-config-show', + 'nix3-copy', + 'nix3-daemon', + 'nix3-derivation-add', + 'nix3-derivation', + 'nix3-derivation-show', + 'nix3-develop', + 'nix3-doctor', + 'nix3-edit', + 'nix3-eval', + 'nix3-flake-archive', + 'nix3-flake-check', + 'nix3-flake-clone', + 'nix3-flake-info', + 'nix3-flake-init', + 'nix3-flake-lock', + 'nix3-flake', + 'nix3-flake-metadata', + 'nix3-flake-new', + 'nix3-flake-prefetch', + 'nix3-flake-show', + 'nix3-flake-update', + 'nix3-fmt', + 'nix3-hash-file', + 'nix3-hash', + 'nix3-hash-path', + 'nix3-hash-to-base16', + 'nix3-hash-to-base32', + 'nix3-hash-to-base64', + 'nix3-hash-to-sri', + 'nix3-help', + 'nix3-help-stores', + 'nix3-key-convert-secret-to-public', + 'nix3-key-generate-secret', + 'nix3-key', + 'nix3-log', + 'nix3-nar-cat', + 'nix3-nar-dump-path', + 'nix3-nar-ls', + 'nix3-nar', + 'nix3-path-info', + 'nix3-print-dev-env', + 'nix3-profile-diff-closures', + 'nix3-profile-history', + 'nix3-profile-install', + 'nix3-profile-list', + 'nix3-profile', + 'nix3-profile-remove', + 'nix3-profile-rollback', + 'nix3-profile-upgrade', + 'nix3-profile-wipe-history', + 'nix3-realisation-info', + 'nix3-realisation', + 'nix3-registry-add', + 'nix3-registry-list', + 'nix3-registry', + 'nix3-registry-pin', + 'nix3-registry-remove', + 'nix3-repl', + 'nix3-run', + 'nix3-search', + 'nix3-shell', + 'nix3-store-add-file', + 'nix3-store-add-path', + 'nix3-store-cat', + 'nix3-store-copy-log', + 'nix3-store-copy-sigs', + 'nix3-store-delete', + 'nix3-store-diff-closures', + 'nix3-store-dump-path', + 'nix3-store-gc', + 'nix3-store-ls', + 'nix3-store-make-content-addressed', + 'nix3-store', + 'nix3-store-optimise', + 'nix3-store-path-from-hash-part', + 'nix3-store-ping', + 'nix3-store-prefetch-file', + 'nix3-store-repair', + 'nix3-store-sign', + 'nix3-store-verify', + 'nix3-upgrade-nix', + 'nix3-why-depends', + 'nix', +] + +foreach page : nix3_manpages + section = '1' + custom_target( + command : [ + bash, + '@INPUT0@', + page, + section, + '@INPUT1@/command-ref/new-cli/@0@.md'.format(page), + '@OUTPUT@', + ], + input : [ + 'render-manpage.sh', + manual_md, + nix, + ], + output : page + '.1', + install : true, + install_dir : get_option('mandir') / 'man1', + ) +endforeach + +nix_manpages = [ + [ 'nix-env', 1 ], + [ 'nix-store', 1 ], + [ 'nix-build', 1 ], + [ 'nix-shell', 1 ], + [ 'nix-instantiate', 1 ], + [ 'nix-collect-garbage', 1 ], + [ 'nix-prefetch-url', 1 ], + [ 'nix-channel', 1 ], + [ 'nix-hash', 1 ], + [ 'nix-copy-closure', 1 ], + [ 'nix.conf', 5, 'conf-file.md' ], + [ 'nix-daemon', 8 ], + [ 'nix-profiles', 5, 'files/profiles.md' ], +] + +foreach entry : nix_manpages + title = entry[0] + # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md, + # rather than a stem identical to its mdbook source. + # Therefore we use an optional third element of this array to override the name pattern + md_file = entry.get(2, title + '.md') + section = entry[1].to_string() + custom_target( + command : [ + bash, + '@INPUT0@', + title, + section, + '@INPUT1@/command-ref/@0@'.format(md_file), + '@OUTPUT@', + ], + input : [ + 'render-manpage.sh', + manual_md, + entry.get(3, []), + nix, + ], + output : '@0@.@1@'.format(entry[0], entry[1]), + install : true, + install_dir : get_option('mandir') / 'man@0@'.format(entry[1]), + ) +endforeach diff --git a/doc/manual/package.nix b/doc/manual/package.nix new file mode 100644 index 000000000..5fd23068c --- /dev/null +++ b/doc/manual/package.nix @@ -0,0 +1,61 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, lowdown +, mdbook +, mdbook-linkcheck +, jq +, nix-ng + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ./.version + ./meson.build + ../../.version + ../../doc/manual + ]; + + # Hack for sake of the dev shell + passthru.baseNativeBuildInputs = [ + meson + ninja + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + jq + ]; + + nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ + nix-ng + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/doc/manual/render-manpage.sh b/doc/manual/render-manpage.sh new file mode 100755 index 000000000..65a9c124e --- /dev/null +++ b/doc/manual/render-manpage.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +lowdown_args= + +if [ "$1" = --out-no-smarty ]; then + lowdown_args=--out-no-smarty + shift +fi + +[ "$#" = 4 ] || { + echo "wrong number of args passed" >&2 + exit 1 +} + +title="$1" +section="$2" +infile="$3" +outfile="$4" + +( + printf "Title: %s\n\n" "$title" + cat "$infile" +) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile" diff --git a/doc/manual/src/command-ref/meson.build b/doc/manual/src/command-ref/meson.build new file mode 100644 index 000000000..bc10b3bbf --- /dev/null +++ b/doc/manual/src/command-ref/meson.build @@ -0,0 +1,67 @@ +xp_features_json = custom_target( + command : [nix, '__dump-xp-features'], + capture : true, + output : 'xp-features.json', +) + +experimental_features_shortlist_md = custom_target( + command : nix_eval_for_docs + [ + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))', + ], + input : [ + '../../generate-xp-features-shortlist.nix', + xp_features_json, + ], + output : 'experimental-features-shortlist.md', + env : nix_env_for_docs, +) + +# Intermediate step for manpage generation. +# This splorks the output of generate-manpage.nix as JSON, +# which gets written as a directory tree below. +nix3_cli_files_json = custom_target( + command : nix_eval_for_docs_common + [ + '--json', + '--expr', + 'import @INPUT0@ true (builtins.readFile @INPUT1@)', + ], + input : [ + '../../generate-manpage.nix', + nix3_cli_json, + ], + capture : true, + output : 'new-cli.json', + env : nix_env_for_docs, +) +nix3_cli_files = custom_target( + command : [ + python, + '@INPUT0@', + '-i', '@INPUT1@', + '-o', '@OUTPUT@', + ], + input : [ + '../../json-to-tree.py', + nix3_cli_files_json, + ], + output : 'new-cli', +) + +conf_file_md = custom_target( + command : [ + nix_eval_for_docs, + '--write-to', '@OUTPUT@', + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))', + ], + capture : true, + input : [ + '../../generate-settings.nix', + '../../utils.nix', + conf_file_json, + experimental_features_shortlist_md, + ], + output : 'conf-file.md', + env : nix_env_for_docs, +) diff --git a/doc/manual/src/contributing/meson.build b/doc/manual/src/contributing/meson.build new file mode 100644 index 000000000..2929578c8 --- /dev/null +++ b/doc/manual/src/contributing/meson.build @@ -0,0 +1,15 @@ +# Intermediate step for experimental-feature-descriptions.md. +# This splorks the output of generate-xp-features.nix as JSON, +# which gets written as a directory tree below. +experimental_feature_descriptions_md = custom_target( + command : nix_eval_for_docs + [ + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))', + ], + input : [ + '../../generate-xp-features.nix', + xp_features_json, + ], + capture : true, + output : 'experimental-feature-descriptions.md', +) diff --git a/doc/manual/src/language/meson.build b/doc/manual/src/language/meson.build new file mode 100644 index 000000000..3b00226e3 --- /dev/null +++ b/doc/manual/src/language/meson.build @@ -0,0 +1,13 @@ +builtins_md = custom_target( + command : nix_eval_for_docs + [ + '--write-to', '@OUTPUT@', + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))', + ], + input : [ + '../../generate-builtins.nix', + language_json, + ], + output : 'builtins.md', + env : nix_env_for_docs, +) diff --git a/doc/manual/src/release-notes/meson.build b/doc/manual/src/release-notes/meson.build new file mode 100644 index 000000000..d9ce7bdda --- /dev/null +++ b/doc/manual/src/release-notes/meson.build @@ -0,0 +1,22 @@ +rl_next_generated = custom_target( + command : [ + 'bash', + '-euo', + 'pipefail', + '-c', + ''' + if type -p build-release-notes > /dev/null; then + build-release-notes --change-authors @CURRENT_SOURCE_DIR@/../../change-authors.yml @CURRENT_SOURCE_DIR@/../../rl-next + fi + @0@ @INPUT0@ @CURRENT_SOURCE_DIR@/../../rl-next > @DEPFILE@ + '''.format( + python.full_path(), + ), + ], + input : [ + generate_manual_deps, + ], + output : 'rl-next-generated.md', + capture : true, + depfile : 'rl-next.d', +) diff --git a/doc/manual/src/store/meson.build b/doc/manual/src/store/meson.build new file mode 100644 index 000000000..a05123efb --- /dev/null +++ b/doc/manual/src/store/meson.build @@ -0,0 +1,13 @@ +types_dir = custom_target( + command : nix_eval_for_docs + [ + '--write-to', '@OUTPUT@', + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@)).stores', + ], + input : [ + '../../generate-store-types.nix', + nix3_cli_json, + ], + output : 'types', + env : nix_env_for_docs, +) diff --git a/flake.nix b/flake.nix index 5ca9c1a45..51cb70ab2 100644 --- a/flake.nix +++ b/flake.nix @@ -221,6 +221,7 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix-ng; + nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; } @@ -349,6 +350,7 @@ ++ pkgs.nixComponents.nix-store.nativeBuildInputs ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.baseNativeBuildInputs ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs diff --git a/meson.build b/meson.build index 8dd44cc10..636d38b08 100644 --- a/meson.build +++ b/meson.build @@ -23,6 +23,9 @@ subproject('nix') # Docs subproject('internal-api-docs') subproject('external-api-docs') +if not meson.is_cross_build() + subproject('nix-manual') +endif # External C wrapper libraries subproject('libutil-c') diff --git a/packaging/components.nix b/packaging/components.nix index 5fc3236cf..a2e32ea94 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -60,6 +60,7 @@ in nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; + nix-manual = callPackage ../src/nix-manual/package.nix { version = fineVersion; }; nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; diff --git a/packaging/everything.nix b/packaging/everything.nix index 6dae7f1c2..7ce7bf5fe 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -33,6 +33,7 @@ nix-functional-tests, + nix-manual, nix-internal-api-docs, nix-external-api-docs, @@ -70,6 +71,7 @@ nix-cli + nix-manual nix-internal-api-docs nix-external-api-docs diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 65978835c..cba1b2583 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -146,6 +146,9 @@ in withCoverageChecks = true; }; + # Nix's manual + manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual; + # API docs for Nix's unstable internal C++ interfaces. internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; diff --git a/src/nix-manual b/src/nix-manual new file mode 120000 index 000000000..492c97408 --- /dev/null +++ b/src/nix-manual @@ -0,0 +1 @@ +../doc/manual/ \ No newline at end of file