Meson the manual

Co-Authored-By: Qyriad <qyriad@qyriad.me>
Co-Authored-By: eldritch horrors <pennae@lix.systems>

doc/manual/.version

@@ -0,0 +1 @@
+../../.version

doc/manual/generate-deps.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import glob
+import sys
+
+# meson expects makefile-style dependency declarations, i.e.
+#
+#   target: dependency...
+#
+# meson seems to pass depfiles straight on to ninja even though
+# it also parses the file itself (or at least has code to do so
+# in its tree), so we must live by ninja's rules: only slashes,
+# spaces and octothorpes can be escaped, anything else is taken
+# literally. since the rules for these aren't even the same for
+# all three we will just fail when we encounter any of them (if
+# asserts are off for some reason the depfile will likely point
+# to nonexistant paths, making everything phony and thus fine.)
+for path in glob.glob(sys.argv[1] + '/**', recursive=True):
+    assert '\\' not in path
+    assert ' ' not in path
+    assert '#' not in path
+    print("ignored:", path)

doc/manual/json-to-tree.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+"""
+This script is a helper for this project's Meson buildsystem, to replace its
+usage of `nix eval --write-to`. Writing a JSON object as a nested directory
+tree is more generic, easier to maintain, and far, far less cursed. Nix
+has 'good' support for JSON output. Let's just use it.
+"""
+
+import argparse
+from pathlib import Path
+import json
+import sys
+
+name = 'json-to-tree.py'
+
+def log(*args, **kwargs):
+    kwargs['file'] = sys.stderr
+    return print(f'{name}:', *args, **kwargs)
+
+def write_dict_to_directory(current_directory: Path, data: dict, files_written=0):
+    current_directory.mkdir(parents=True, exist_ok=True)
+    for key, value in data.items():
+        nested_path = current_directory / key
+        match value:
+            case dict(nested_data):
+                files_written += write_dict_to_directory(nested_path, nested_data)
+
+            case str(content):
+                nested_path.write_text(content)
+                files_written += 1
+
+            case rest:
+                assert False, \
+                    f'should have been called on a dict or string, not {type(rest)=}\n\t{rest=}'
+
+    return files_written
+
+def main():
+    parser = argparse.ArgumentParser(name)
+    parser.add_argument('-i', '--input', type=argparse.FileType('r'), default='-',
+        help='The JSON input to operate on and output as a directory tree',
+    )
+    parser.add_argument('-o', '--output', type=Path, required=True,
+        help='The place to put the directory tree',
+    )
+    args = parser.parse_args()
+
+    json_string = args.input.read()
+
+    try:
+        data = json.loads(json_string)
+    except json.JSONDecodeError:
+        log(f'could not decode JSON from input: {json_string}')
+        raise
+
+
+    files_written = write_dict_to_directory(args.output, data)
+    log(f'wrote {files_written} files')
+
+sys.exit(main())

doc/manual/meson.build

@@ -0,0 +1,371 @@
+project('nix-manual',
+  version : files('.version'),
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+nix = find_program('nix', native : true)
+
+mdbook = find_program('mdbook', native : true)
+bash = find_program('bash', native : true)
+
+pymod = import('python')
+python = pymod.find_installation('python3')
+
+nix_env_for_docs = {
+  'HOME': '/dummy',
+  'NIX_CONF_DIR': '/dummy',
+  'NIX_SSL_CERT_FILE': '/dummy/no-ca-bundle.crt',
+  'NIX_STATE_DIR': '/dummy',
+  'NIX_CONFIG': 'cores = 0',
+}
+
+nix_for_docs = [ nix, '--experimental-features', 'nix-command' ]
+nix_eval_for_docs_common = nix_for_docs + [
+  'eval',
+  '-I', 'nix=' + meson.current_source_dir(),
+  '--store', 'dummy://',
+  '--impure',
+]
+nix_eval_for_docs = nix_eval_for_docs_common + '--raw'
+
+conf_file_json = custom_target(
+  command : nix_for_docs + [ 'show-config', '--json' ],
+  capture : true,
+  output : 'conf-file.json',
+  env : nix_env_for_docs,
+)
+
+nix_conf_file_md_body = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT1@ { prefix = "conf"; } (builtins.fromJSON (builtins.readFile ./@INPUT0@))',
+  ],
+  capture : true,
+  input : [
+    conf_file_json,
+    'utils.nix',
+    'generate-settings.nix',
+    'src' / 'command-ref' / 'conf-file-prefix.md',
+    'src' / 'command-ref' / 'experimental-features-shortlist.md'
+  ],
+  output : 'conf-file.md.body',
+  env : nix_env_for_docs,
+)
+
+nix_conf_file_md = custom_target(
+  command : [ 'cat', '@INPUT@' ],
+  capture : true,
+  input : [
+    'src/command-ref/conf-file-prefix.md',
+    nix_conf_file_md_body,
+  ],
+  output : 'conf-file.md',
+)
+
+nix_exp_features_json = custom_target(
+  command : [ nix, '__dump-xp-features' ],
+  capture : true,
+  output : 'xp-features.json',
+)
+
+language_json = custom_target(
+  command: [nix, '__dump-language'],
+  output : 'language.json',
+  capture : true,
+  env : nix_env_for_docs,
+)
+
+nix3_cli_json = custom_target(
+  command : [ nix, '__dump-cli' ],
+  capture : true,
+  output : 'nix.json',
+  env : nix_env_for_docs,
+)
+
+generate_manual_deps = files(
+  'generate-deps.py',
+)
+
+# Generates types
+subdir('src/store')
+# Generates builtins.md and builtin-constants.md.
+subdir('src/language')
+# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md.
+subdir('src/command-ref')
+# Generates experimental-feature-descriptions.md.
+subdir('src/contributing')
+# Generates rl-next-generated.md.
+subdir('src/release-notes')
+
+manual = custom_target(
+  'manual',
+  command : [
+    bash,
+    '-euo', 'pipefail',
+    '-c',
+    '''
+        @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
+        cd @SOURCE_ROOT@
+        @1@ build doc/manual -d @2@ | { grep -Fv "because fragment resolution isn't implemented" || :; }
+        rm -rf @2@/manual
+        mv @2@/html @2@/manual
+        find @2@/manual -iname meson.build -delete
+    '''.format(
+      python.full_path(),
+      mdbook.full_path(),
+      meson.current_build_dir(),
+    ),
+  ],
+  input : [
+    generate_manual_deps,
+    'book.toml',
+    'anchors.jq',
+    'custom.css',
+    nix3_cli_files,
+    experimental_features_shortlist_md,
+    experimental_feature_descriptions_md,
+    conf_file_md,
+    builtins_md,
+    rl_next_generated,
+    nix,
+  ],
+  output : [
+    'manual',
+    'markdown',
+  ],
+  depfile : 'manual.d',
+  env : {
+    'RUST_LOG': 'info',
+    'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src',
+  },
+)
+manual_html = manual[0]
+manual_md = manual[1]
+
+install_subdir(
+  manual_html.full_path(),
+  install_dir : get_option('datadir') / 'doc/nix',
+)
+
+nix_nested_manpages = [
+  [ 'nix-env',
+    [
+      'delete-generations',
+      'install',
+      'list-generations',
+      'query',
+      'rollback',
+      'set-flag',
+      'set',
+      'switch-generation',
+      'switch-profile',
+      'uninstall',
+      'upgrade',
+    ],
+  ],
+  [ 'nix-store',
+    [
+      'add-fixed',
+      'add',
+      'delete',
+      'dump-db',
+      'dump',
+      'export',
+      'gc',
+      'generate-binary-cache-key',
+      'import',
+      'load-db',
+      'optimise',
+      'print-env',
+      'query',
+      'read-log',
+      'realise',
+      'repair-path',
+      'restore',
+      'serve',
+      'verify',
+      'verify-path',
+    ],
+  ],
+]
+
+foreach command : nix_nested_manpages
+  foreach page : command[1]
+    title = command[0] + ' --' + page
+    section = '1'
+    custom_target(
+      command : [
+        './render-manpage.sh',
+        '--out-no-smarty',
+        title,
+        section,
+        '@INPUT0@/command-ref' / command[0] / (page + '.md'),
+        '@OUTPUT0@',
+      ],
+      input : [
+        manual_md,
+        nix,
+      ],
+      output : command[0] + '-' + page + '.1',
+      install : true,
+      install_dir : get_option('mandir') / 'man1',
+    )
+  endforeach
+endforeach
+
+nix3_manpages = [
+  'nix3-build',
+  'nix3-bundle',
+  'nix3-config',
+  'nix3-config-show',
+  'nix3-copy',
+  'nix3-daemon',
+  'nix3-derivation-add',
+  'nix3-derivation',
+  'nix3-derivation-show',
+  'nix3-develop',
+  'nix3-doctor',
+  'nix3-edit',
+  'nix3-eval',
+  'nix3-flake-archive',
+  'nix3-flake-check',
+  'nix3-flake-clone',
+  'nix3-flake-info',
+  'nix3-flake-init',
+  'nix3-flake-lock',
+  'nix3-flake',
+  'nix3-flake-metadata',
+  'nix3-flake-new',
+  'nix3-flake-prefetch',
+  'nix3-flake-show',
+  'nix3-flake-update',
+  'nix3-fmt',
+  'nix3-hash-file',
+  'nix3-hash',
+  'nix3-hash-path',
+  'nix3-hash-to-base16',
+  'nix3-hash-to-base32',
+  'nix3-hash-to-base64',
+  'nix3-hash-to-sri',
+  'nix3-help',
+  'nix3-help-stores',
+  'nix3-key-convert-secret-to-public',
+  'nix3-key-generate-secret',
+  'nix3-key',
+  'nix3-log',
+  'nix3-nar-cat',
+  'nix3-nar-dump-path',
+  'nix3-nar-ls',
+  'nix3-nar',
+  'nix3-path-info',
+  'nix3-print-dev-env',
+  'nix3-profile-diff-closures',
+  'nix3-profile-history',
+  'nix3-profile-install',
+  'nix3-profile-list',
+  'nix3-profile',
+  'nix3-profile-remove',
+  'nix3-profile-rollback',
+  'nix3-profile-upgrade',
+  'nix3-profile-wipe-history',
+  'nix3-realisation-info',
+  'nix3-realisation',
+  'nix3-registry-add',
+  'nix3-registry-list',
+  'nix3-registry',
+  'nix3-registry-pin',
+  'nix3-registry-remove',
+  'nix3-repl',
+  'nix3-run',
+  'nix3-search',
+  'nix3-shell',
+  'nix3-store-add-file',
+  'nix3-store-add-path',
+  'nix3-store-cat',
+  'nix3-store-copy-log',
+  'nix3-store-copy-sigs',
+  'nix3-store-delete',
+  'nix3-store-diff-closures',
+  'nix3-store-dump-path',
+  'nix3-store-gc',
+  'nix3-store-ls',
+  'nix3-store-make-content-addressed',
+  'nix3-store',
+  'nix3-store-optimise',
+  'nix3-store-path-from-hash-part',
+  'nix3-store-ping',
+  'nix3-store-prefetch-file',
+  'nix3-store-repair',
+  'nix3-store-sign',
+  'nix3-store-verify',
+  'nix3-upgrade-nix',
+  'nix3-why-depends',
+  'nix',
+]
+
+foreach page : nix3_manpages
+  section = '1'
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      page,
+      section,
+      '@INPUT1@/command-ref/new-cli/@0@.md'.format(page),
+      '@OUTPUT@',
+    ],
+    input : [
+      'render-manpage.sh',
+      manual_md,
+      nix,
+    ],
+    output : page + '.1',
+    install : true,
+    install_dir : get_option('mandir') / 'man1',
+  )
+endforeach
+
+nix_manpages = [
+  [ 'nix-env', 1 ],
+  [ 'nix-store', 1 ],
+  [ 'nix-build', 1 ],
+  [ 'nix-shell', 1 ],
+  [ 'nix-instantiate', 1 ],
+  [ 'nix-collect-garbage', 1 ],
+  [ 'nix-prefetch-url', 1 ],
+  [ 'nix-channel', 1 ],
+  [ 'nix-hash', 1 ],
+  [ 'nix-copy-closure', 1 ],
+  [ 'nix.conf', 5, 'conf-file.md' ],
+  [ 'nix-daemon', 8 ],
+  [ 'nix-profiles', 5, 'files/profiles.md' ],
+]
+
+foreach entry : nix_manpages
+  title = entry[0]
+  # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md,
+  # rather than a stem identical to its mdbook source.
+  # Therefore we use an optional third element of this array to override the name pattern
+  md_file = entry.get(2, title + '.md')
+  section = entry[1].to_string()
+  custom_target(
+    command : [
+      bash,
+      '@INPUT0@',
+      title,
+      section,
+      '@INPUT1@/command-ref/@0@'.format(md_file),
+      '@OUTPUT@',
+    ],
+    input : [
+      'render-manpage.sh',
+      manual_md,
+      entry.get(3, []),
+      nix,
+    ],
+    output : '@0@.@1@'.format(entry[0], entry[1]),
+    install : true,
+    install_dir : get_option('mandir') / 'man@0@'.format(entry[1]),
+  )
+endforeach

doc/manual/package.nix

@@ -0,0 +1,61 @@
+{ lib
+, mkMesonDerivation
+
+, meson
+, ninja
+, lowdown
+, mdbook
+, mdbook-linkcheck
+, jq
+, nix-ng
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-internal-api-docs";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ./.version
+    ./meson.build
+    ../../.version
+    ../../doc/manual
+  ];
+
+  # Hack for sake of the dev shell
+  passthru.baseNativeBuildInputs = [
+    meson
+    ninja
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+    jq
+  ];
+
+  nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [
+    nix-ng
+  ];
+
+  preConfigure =
+    ''
+      chmod u+w ./.version
+      echo ${finalAttrs.version} > ./.version
+    '';
+
+  postInstall = ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  meta = {
+    platforms = lib.platforms.all;
+  };
+})

doc/manual/render-manpage.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+lowdown_args=
+
+if [ "$1" = --out-no-smarty ]; then
+    lowdown_args=--out-no-smarty
+    shift
+fi
+
+[ "$#" = 4 ] || {
+    echo "wrong number of args passed" >&2
+    exit 1
+}
+
+title="$1"
+section="$2"
+infile="$3"
+outfile="$4"
+
+(
+    printf "Title: %s\n\n" "$title"
+    cat "$infile"
+) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile"

doc/manual/src/command-ref/meson.build

@@ -0,0 +1,67 @@
+xp_features_json = custom_target(
+  command : [nix, '__dump-xp-features'],
+  capture : true,
+  output : 'xp-features.json',
+)
+
+experimental_features_shortlist_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features-shortlist.nix',
+    xp_features_json,
+  ],
+  output : 'experimental-features-shortlist.md',
+  env : nix_env_for_docs,
+)
+
+# Intermediate step for manpage generation.
+# This splorks the output of generate-manpage.nix as JSON,
+# which gets written as a directory tree below.
+nix3_cli_files_json = custom_target(
+  command : nix_eval_for_docs_common + [
+    '--json',
+    '--expr',
+    'import @INPUT0@ true (builtins.readFile @INPUT1@)',
+  ],
+  input : [
+    '../../generate-manpage.nix',
+    nix3_cli_json,
+  ],
+  capture : true,
+  output : 'new-cli.json',
+  env : nix_env_for_docs,
+)
+nix3_cli_files = custom_target(
+  command : [
+    python,
+    '@INPUT0@',
+    '-i', '@INPUT1@',
+    '-o', '@OUTPUT@',
+  ],
+  input : [
+    '../../json-to-tree.py',
+    nix3_cli_files_json,
+  ],
+  output : 'new-cli',
+)
+
+conf_file_md = custom_target(
+  command : [
+    nix_eval_for_docs,
+    '--write-to', '@OUTPUT@',
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  capture : true,
+  input : [
+    '../../generate-settings.nix',
+    '../../utils.nix',
+    conf_file_json,
+    experimental_features_shortlist_md,
+  ],
+  output : 'conf-file.md',
+  env : nix_env_for_docs,
+)

doc/manual/src/contributing/meson.build

@@ -0,0 +1,15 @@
+# Intermediate step for experimental-feature-descriptions.md.
+# This splorks the output of generate-xp-features.nix as JSON,
+# which gets written as a directory tree below.
+experimental_feature_descriptions_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features.nix',
+    xp_features_json,
+  ],
+  capture : true,
+  output : 'experimental-feature-descriptions.md',
+)

doc/manual/src/language/meson.build

@@ -0,0 +1,13 @@
+builtins_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--write-to', '@OUTPUT@',
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  input : [
+    '../../generate-builtins.nix',
+    language_json,
+  ],
+  output : 'builtins.md',
+  env : nix_env_for_docs,
+)

doc/manual/src/release-notes/meson.build

@@ -0,0 +1,22 @@
+rl_next_generated = custom_target(
+  command : [
+    'bash',
+    '-euo',
+    'pipefail',
+    '-c',
+    '''
+      if type -p build-release-notes > /dev/null; then
+        build-release-notes --change-authors @CURRENT_SOURCE_DIR@/../../change-authors.yml @CURRENT_SOURCE_DIR@/../../rl-next
+      fi
+      @0@ @INPUT0@ @CURRENT_SOURCE_DIR@/../../rl-next > @DEPFILE@
+    '''.format(
+      python.full_path(),
+    ),
+  ],
+  input : [
+    generate_manual_deps,
+  ],
+  output : 'rl-next-generated.md',
+  capture : true,
+  depfile : 'rl-next.d',
+)

doc/manual/src/store/meson.build

@@ -0,0 +1,13 @@
+types_dir = custom_target(
+  command : nix_eval_for_docs + [
+    '--write-to', '@OUTPUT@',
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@)).stores',
+  ],
+  input : [
+    '../../generate-store-types.nix',
+    nix3_cli_json,
+  ],
+  output : 'types',
+  env : nix_env_for_docs,
+)

flake.nix

@@ -221,6 +221,7 @@
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
           default = self.packages.${system}.nix-ng;
+          nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual;
           nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
           nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
         }
@@ -349,6 +350,7 @@
             ++ pkgs.nixComponents.nix-store.nativeBuildInputs
             ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
             ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
+            ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.baseNativeBuildInputs
             ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
             ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
             ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs

meson.build

@@ -23,6 +23,9 @@ subproject('nix')
 # Docs
 subproject('internal-api-docs')
 subproject('external-api-docs')
+if not meson.is_cross_build()
+  subproject('nix-manual')
+endif
 
 # External C wrapper libraries
 subproject('libutil-c')

packaging/components.nix

@@ -60,6 +60,7 @@ in
 
   nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; };
 
+  nix-manual = callPackage ../src/nix-manual/package.nix { version = fineVersion; };
   nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; };
   nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; };
 

packaging/everything.nix

@@ -33,6 +33,7 @@
 
   nix-functional-tests,
 
+  nix-manual,
   nix-internal-api-docs,
   nix-external-api-docs,
 
@@ -70,6 +71,7 @@
 
     nix-cli
 
+    nix-manual
     nix-internal-api-docs
     nix-external-api-docs
 

packaging/hydra.nix

@@ -146,6 +146,9 @@ in
     withCoverageChecks = true;
   };
 
+  # Nix's manual
+  manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual;
+
   # API docs for Nix's unstable internal C++ interfaces.
   internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs;
 

src/nix-manual

@@ -0,0 +1 @@
+../doc/manual/