From d599b4f77367a6c4e841688f51ce973cb93cf9a3 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 16 Sep 2024 00:24:16 +0200 Subject: [PATCH 1/3] Introduce `nix derivation instantiate` This is simmilar to `nix-instantiate` with support for installables (and flakes). --- src/nix/derivation-instantiate.cc | 93 ++++++++++++++++++++++ src/nix/derivation-instantiate.md | 39 +++++++++ src/nix/meson.build | 1 + tests/functional/derivation-instantiate.sh | 44 ++++++++++ tests/functional/local.mk | 1 + tests/functional/meson.build | 1 + 6 files changed, 179 insertions(+) create mode 100644 src/nix/derivation-instantiate.cc create mode 100644 src/nix/derivation-instantiate.md create mode 100755 tests/functional/derivation-instantiate.sh diff --git a/src/nix/derivation-instantiate.cc b/src/nix/derivation-instantiate.cc new file mode 100644 index 000000000..9e2f40c04 --- /dev/null +++ b/src/nix/derivation-instantiate.cc @@ -0,0 +1,93 @@ +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "local-fs-store.hh" +#include "progress-bar.hh" + +#include + +using namespace nix; + +static nlohmann::json storePathSetToJSON(const StorePathSet & paths, Store & store) +{ + auto res = nlohmann::json::object(); + for (auto & path : paths) { + res[store.printStorePath(path)] = nlohmann::json::object(); + } + return res; +} + +// TODO deduplicate with other code also setting such out links. +static void +createOutLinks(const std::filesystem::path & outLink, const StorePathSet & derivations, LocalFSStore & store) +{ + for (const auto & [_i, drv] : enumerate(derivations)) { + auto i = _i; + auto symlink = outLink; + + if (i) + symlink += fmt("-%d", i); + store.addPermRoot(drv, absPath(symlink.string())); + } +} + +struct CmdDerivationInstantiate : InstallablesCommand, MixJSON +{ + Path outLink = "drv"; + bool printOutputPaths = false; + + CmdDerivationInstantiate() + { + addFlag( + {.longName = "out-link", + .shortName = 'o', + .description = "Use *path* as prefix for the symlinks to the evaluation results. It defaults to `drv`.", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath}); + + addFlag({ + .longName = "no-link", + .description = "Do not create symlinks to the evaluation results.", + .handler = {&outLink, Path("")}, + }); + } + + std::string description() override + { + return "Force the evaluation of the expression and return the corresponding .drv"; + } + + std::string doc() override + { + return +#include "derivation-instantiate.md" + ; + } + + Category category() override + { + return catSecondary; + } + + void run(ref store, Installables && installables) override + { + auto drvPaths = Installable::toDerivations(store, installables, false); + + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast()) + createOutLinks(outLink, drvPaths, *store2); + + if (json) { + logger->cout("%s", storePathSetToJSON(drvPaths, *store).dump()); + } else { + stopProgressBar(); + for (auto & path : drvPaths) { + logger->cout(store->printStorePath(path)); + } + } + } +}; + +static auto rCmdDerivationInstantiate = registerCommand2({"derivation", "instantiate"}); diff --git a/src/nix/derivation-instantiate.md b/src/nix/derivation-instantiate.md new file mode 100644 index 000000000..8470e06f8 --- /dev/null +++ b/src/nix/derivation-instantiate.md @@ -0,0 +1,39 @@ +R""( + +# Name + +`nix derivation instantiate` - instantiate store derivations + +# Synopsis + +`nix derivation instantiate` + [`--out-link` *link prefix*] + [`--json`] + [`--no-link`] + *installables…* + +# Description + +The command `nix derivation instantiate` produces [store derivation]s from +installables. Each top-level expression should evaluate to a derivation, a list +of derivations, or a set of derivations. The paths of the resulting store +derivations are printed on standard output. + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +# Options + +- `--out-link` *link prefix* + + The prefix used for gc roots. + +- `--no-link` + + Do not create garbage collector roots for the generated store derivations. + +- `--json` + + Dump a JSON object whose keys are the generated store derivations instread of + printing them directly on the output. + +)"" diff --git a/src/nix/meson.build b/src/nix/meson.build index 6edb768e3..95814c7ff 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -78,6 +78,7 @@ nix_sources = [config_h] + files( 'config.cc', 'copy.cc', 'derivation-add.cc', + 'derivation-instantiate.cc', 'derivation-show.cc', 'derivation.cc', 'develop.cc', diff --git a/tests/functional/derivation-instantiate.sh b/tests/functional/derivation-instantiate.sh new file mode 100755 index 000000000..b46b177bc --- /dev/null +++ b/tests/functional/derivation-instantiate.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +drvPath=$(nix derivation instantiate --no-link --file simple.nix) +test -f "$drvPath" +nix-store --delete "$drvPath" +if test -f "$drvPath"; then false; fi + +drvPath=$(nix derivation instantiate --file simple.nix) +test -f "$drvPath" +test -e drv +nix-store --gc --print-roots | grep "$drvPath" +nix-store --gc --print-live | grep "$drvPath" +if nix-store --delete "$drvPath"; then false; fi +test -f "$drvPath" +[ "$(nix-store -q --roots "$drvPath")" = "$(realpath --no-symlinks drv) -> $drvPath" ] +rm drv +nix-store --delete "$drvPath" +if test -f "$drvPath"; then false; fi + +drvPath=$(nix derivation instantiate --out-link foobar --file simple.nix) +test -e foobar +[ "$(nix-store -q --roots "$drvPath")" = "$(realpath --no-symlinks foobar) -> $drvPath" ] +rm foobar +nix-store --delete "$drvPath" + +drvPathJson=$(nix derivation instantiate --json --no-link --file simple.nix) +[ "$drvPathJson" = "{\"$drvPath\":{}}" ] +nix-store --delete "$drvPath" + +mapfile -t drvPaths < <(nix derivation instantiate --json --out-link multidrv --file check.nix | jq 'keys|.[]' -r) +roots=(./multidrv*) +[ "${#roots[@]}" -gt 1 ] +[ "${#roots[@]}" -eq "${#drvPaths[@]}" ] +mapfile -t rootedPaths < <(readlink "${roots[@]}") +[ "${rootedPaths[*]}" = "${drvPaths[*]}" ] +rm -f multidrv* + +nix-collect-garbage diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 3f796291a..3f472ce21 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -88,6 +88,7 @@ nix_tests = \ why-depends.sh \ derivation-json.sh \ derivation-advanced-attributes.sh \ + derivation-instantiate.sh \ import-from-derivation.sh \ nix_path.sh \ nars.sh \ diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 69b6d3194..4188386a9 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -157,6 +157,7 @@ suites = [ 'why-depends.sh', 'derivation-json.sh', 'derivation-advanced-attributes.sh', + 'derivation-instantiate.sh', 'import-from-derivation.sh', 'nix_path.sh', 'nars.sh', From 3edc164a86c0156f7e0bd46305aeacb352a33e5a Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 7 Oct 2024 20:38:53 +0200 Subject: [PATCH 2/3] Change, document and test output order --- src/libcmd/installables.cc | 8 ++-- src/libcmd/installables.hh | 2 +- src/nix/derivation-instantiate.cc | 10 +++-- src/nix/derivation-instantiate.md | 50 +++++++++++++++++++++- src/nix/derivation-show.cc | 3 +- tests/functional/derivation-instantiate.sh | 20 ++++++++- 6 files changed, 79 insertions(+), 14 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f9d6d8ce8..83908e44c 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -803,18 +803,18 @@ StorePath Installable::toStorePath( return *paths.begin(); } -StorePathSet Installable::toDerivations( +StorePaths Installable::toDerivations( ref store, const Installables & installables, bool useDeriver) { - StorePathSet drvPaths; + StorePaths drvPaths; for (const auto & i : installables) for (const auto & b : i->toDerivedPaths()) std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - drvPaths.insert( + drvPaths.push_back( bo.path.isDerivation() ? bo.path : useDeriver @@ -822,7 +822,7 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); + drvPaths.push_back(resolveDerivedPath(*store, *bfd.drvPath)); }, }, b.path.raw()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index bf5759230..9fe2702e6 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -186,7 +186,7 @@ struct Installable OperateOn operateOn, ref installable); - static std::set toDerivations( + static std::vector toDerivations( ref store, const Installables & installables, bool useDeriver = false); diff --git a/src/nix/derivation-instantiate.cc b/src/nix/derivation-instantiate.cc index 9e2f40c04..5c3caf060 100644 --- a/src/nix/derivation-instantiate.cc +++ b/src/nix/derivation-instantiate.cc @@ -9,18 +9,20 @@ using namespace nix; -static nlohmann::json storePathSetToJSON(const StorePathSet & paths, Store & store) +static nlohmann::json storePathSetToJSON(const StorePaths & paths, Store & store) { - auto res = nlohmann::json::object(); + nlohmann::json res; for (auto & path : paths) { - res[store.printStorePath(path)] = nlohmann::json::object(); + nlohmann::json entry; + entry["drvPath"] = store.printStorePath(path); + res.push_back(entry); } return res; } // TODO deduplicate with other code also setting such out links. static void -createOutLinks(const std::filesystem::path & outLink, const StorePathSet & derivations, LocalFSStore & store) +createOutLinks(const std::filesystem::path & outLink, const StorePaths & derivations, LocalFSStore & store) { for (const auto & [_i, drv] : enumerate(derivations)) { auto i = _i; diff --git a/src/nix/derivation-instantiate.md b/src/nix/derivation-instantiate.md index 8470e06f8..465f26265 100644 --- a/src/nix/derivation-instantiate.md +++ b/src/nix/derivation-instantiate.md @@ -33,7 +33,53 @@ derivations are printed on standard output. - `--json` - Dump a JSON object whose keys are the generated store derivations instread of - printing them directly on the output. + Dump a JSON list of objects containing at least a `drvPath` field with the + path to the produced store derivation. + +# Examples + +* Get the store derivation for a single installable, with a gc root + + ```console + $ nix derivation instantiate github:NixOS/nixpkgs#hello + /nix/store/af3rc6phyv80h7aq4y3d08awnq2ja8fp-hello-2.12.1.drv + $ ls -ld drv + lrwxrwxrwx [...] drv -> /nix/store/af3rc6phyv80h7aq4y3d08awnq2ja8fp-hello-2.12.1.drv + ``` + +* Get the store derivations for multiple installables, in the same order as the + provided arguments. + + ```console + $ nix derivation instantiate github:NixOS/nixpkgs#{hello,xorg.xclock} + /nix/store/af3rc6phyv80h7aq4y3d08awnq2ja8fp-hello-2.12.1.drv + /nix/store/82w6jak6c7zldgvxyq5nwhclz3yp85zp-xclock-1.1.1.drv + ``` + +* The same, with JSON output. The values also appear in the same order as CLI parameters. + + ```console + $ nix derivation instantiate github:NixOS/nixpkgs#{xorg.xclock,hello} --json | jq + [ + { + "drvPath": "/nix/store/82w6jak6c7zldgvxyq5nwhclz3yp85zp-xclock-1.1.1.drv" + }, + { + "drvPath": "/nix/store/af3rc6phyv80h7aq4y3d08awnq2ja8fp-hello-2.12.1.drv" + } + ] + ``` + +# Notes + +* JSON output format may be extended in the future with other fields. + +* Order guarantees will always ensure that the following bash commands output + the same text. + + ```console + $ nix derivation instantiate [installables] + $ nix derivation instantiate [installables] --json | jq ".[] | .drvPath" -r + ``` )"" diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index bf637246d..dd5df0f82 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -41,7 +41,8 @@ struct CmdShowDerivation : InstallablesCommand void run(ref store, Installables && installables) override { - auto drvPaths = Installable::toDerivations(store, installables, true); + auto drvPathsList = Installable::toDerivations(store, installables, true); + StorePathSet drvPaths(drvPathsList.begin(), drvPathsList.end()); if (recursive) { StorePathSet closure; diff --git a/tests/functional/derivation-instantiate.sh b/tests/functional/derivation-instantiate.sh index b46b177bc..20489e927 100755 --- a/tests/functional/derivation-instantiate.sh +++ b/tests/functional/derivation-instantiate.sh @@ -11,6 +11,7 @@ test -f "$drvPath" nix-store --delete "$drvPath" if test -f "$drvPath"; then false; fi +rm -f drv drvPath=$(nix derivation instantiate --file simple.nix) test -f "$drvPath" test -e drv @@ -23,6 +24,7 @@ rm drv nix-store --delete "$drvPath" if test -f "$drvPath"; then false; fi +rm -f foobar drvPath=$(nix derivation instantiate --out-link foobar --file simple.nix) test -e foobar [ "$(nix-store -q --roots "$drvPath")" = "$(realpath --no-symlinks foobar) -> $drvPath" ] @@ -30,10 +32,11 @@ rm foobar nix-store --delete "$drvPath" drvPathJson=$(nix derivation instantiate --json --no-link --file simple.nix) -[ "$drvPathJson" = "{\"$drvPath\":{}}" ] +[ "$drvPathJson" = "[{\"drvPath\":\"$drvPath\"}]" ] nix-store --delete "$drvPath" -mapfile -t drvPaths < <(nix derivation instantiate --json --out-link multidrv --file check.nix | jq 'keys|.[]' -r) +rm -f multidrv* +mapfile -t drvPaths < <(nix derivation instantiate --json --out-link multidrv --file check.nix | jq '.[]|.drvPath' -r) roots=(./multidrv*) [ "${#roots[@]}" -gt 1 ] [ "${#roots[@]}" -eq "${#drvPaths[@]}" ] @@ -41,4 +44,17 @@ mapfile -t rootedPaths < <(readlink "${roots[@]}") [ "${rootedPaths[*]}" = "${drvPaths[*]}" ] rm -f multidrv* +# The order should always be the same in text and json outputs +jsonOutput=$(nix derivation instantiate --no-link --file check.nix --json | jq '.[]|.drvPath' -r) +textOutput=$(nix derivation instantiate --no-link --file check.nix) +[ "$jsonOutput" = "$textOutput" ] + +# Test that the order is the same as on the command line, and that repeated +# inputs are present several times in the output, in the correct order +nix derivation instantiate --no-link --file multiple-outputs.nix a b a --json | jq --exit-status ' + (.[0].drvPath | match(".*multiple-outputs-a.drv")) + and (.[1].drvPath | match(".*multiple-outputs-b.drv")) + and (.[2].drvPath | match(".*multiple-outputs-a.drv")) +' + nix-collect-garbage From c5ec55c9f5ced49f0999ae7ae55643c2b7bc9cfd Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 8 Oct 2024 01:55:02 +0200 Subject: [PATCH 3/3] nit: clang-tidy --- src/nix/derivation-instantiate.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix/derivation-instantiate.cc b/src/nix/derivation-instantiate.cc index 5c3caf060..30469d258 100644 --- a/src/nix/derivation-instantiate.cc +++ b/src/nix/derivation-instantiate.cc @@ -21,8 +21,7 @@ static nlohmann::json storePathSetToJSON(const StorePaths & paths, Store & store } // TODO deduplicate with other code also setting such out links. -static void -createOutLinks(const std::filesystem::path & outLink, const StorePaths & derivations, LocalFSStore & store) +static void createOutLinks(const std::filesystem::path & outLink, const StorePaths & derivations, LocalFSStore & store) { for (const auto & [_i, drv] : enumerate(derivations)) { auto i = _i;