From d8bfeda1643f434ae015caa4f59d9f4b7f4270d7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 11 May 2023 19:23:13 +0200 Subject: [PATCH 01/30] document identifier syntax for attribute sets it's more likely for readers to find it right there. this also slightly rewords examples to make them stand out better. in the long run there probably needs to be a dedicated section on formal syntax, and better highlighting of examples. --- doc/manual/src/language/constructs.md | 47 ++++++++++++++++++--------- doc/manual/src/language/operators.md | 11 +++---- doc/manual/src/language/values.md | 20 +++++++++--- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index 1c01f2cc7..c53eb8889 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -2,8 +2,11 @@ ## Recursive sets -Recursive sets are just normal sets, but the attributes can refer to -each other. For example, +Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: ```nix rec { @@ -12,7 +15,9 @@ rec { }.x ``` -evaluates to `123`. Note that without `rec` the binding `x = y;` would +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would refer to the variable `y` in the surrounding scope, if one exists, and would be invalid if no such variable exists. That is, in a normal (non-recursive) set, attributes are not added to the lexical scope; in a @@ -33,7 +38,10 @@ will crash with an `infinite recursion encountered` error message. ## Let-expressions A let-expression allows you to define local variables for an expression. -For instance, + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: ```nix let @@ -42,18 +50,19 @@ let in x + y ``` -evaluates to `"foobar"`. +This evaluates to `"foobar"`. ## Inheriting attributes -When defining a set or in a let-expression it is often convenient to -copy variables from the surrounding lexical scope (e.g., when you want -to propagate attributes). This can be shortened using the `inherit` -keyword. For instance, +When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: ```nix let x = 123; in -{ inherit x; +{ + inherit x; y = 456; } ``` @@ -62,15 +71,23 @@ is equivalent to ```nix let x = 123; in -{ x = x; +{ + x = x; y = 456; } ``` -and both evaluate to `{ x = 123; y = 456; }`. (Note that this works -because `x` is added to the lexical scope by the `let` construct.) It is -also possible to inherit attributes from another set. For instance, in -this fragment from `all-packages.nix`, +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, ```nix graphviz = (import ../tools/graphics/graphviz) { diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 3e929724d..f8382ae19 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -35,17 +35,14 @@ ## Attribute selection +> *attrset* `.` *attrpath* \[ `or` *expr* \] + Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. - +An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). -An attribute path is a dot-separated list of attribute names. -An attribute name can be an identifier or a string. - -> *attrpath* = *name* [ `.` *name* ]... \ -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` +> *attrpath* = *name* [ `.` *name* ]... [Attribute selection]: #attribute-selection diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 9d0301753..2ae3e143a 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -164,9 +164,17 @@ Note that lists are only lazy in values, and they are strict in length. An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute name can be an identifier or a [string](#string). +An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). + +> *name* = *identifier* | *string* \ +> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` + Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). +> *attrset* = `{` [ *name* `=` *expr* `;` `]`... `}` + Attributes can appear in any order. An attribute name may only occur once. @@ -182,15 +190,19 @@ Example: This defines a set with attributes named `x`, `text`, `y`. -Attributes can be selected from a set using the `.` operator. For -instance, +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: ```nix { a = "Foo"; b = "Bar"; }.a ``` -evaluates to `"Foo"`. It is possible to provide a default value in an -attribute selection using the `or` keyword: +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: ```nix { a = "Foo"; b = "Bar"; }.c or "Xyzzy" From 549f8dc97b56babb0e23e2c097dce2fa283b6d09 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 11 May 2023 20:04:16 +0200 Subject: [PATCH 02/30] document `builtins.currentTime` --- doc/manual/src/language/builtin-constants.md | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index c6bc9b74c..204daab6e 100644 --- a/doc/manual/src/language/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md @@ -17,3 +17,26 @@ These constants are built into the Nix language evaluator: The built-in value `currentSystem` evaluates to the Nix platform identifier for the Nix installation on which the expression is being evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. + +- [`builtins.currentTime`]{#builtins-currentTime} (integer) + + Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. + Repeated references to that name will re-use the initially obtained value. + + Example: + + ```console + $ nix repl + Welcome to Nix 2.15.1 Type :? for help. + + nix-repl> builtins.currentTime + 1683705525 + + nix-repl> builtins.currentTime + 1683705525 + ``` + + > **Warning** + > + > This is intended for testing and debugging only. + > Do not use it in production systems, since an impure, that is, non-constant value will break reproducibility. From 6a5a8f51bb9773ab4aeac8f508872bcf5ec6fda2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 17 May 2023 14:59:47 +0200 Subject: [PATCH 03/30] add cross-references to pure evaluation mode use consistent wording everywhere. add some details on the configuration option documentation. --- doc/manual/src/language/builtin-constants.md | 9 +++++---- src/libexpr/eval.hh | 8 +++++++- src/libexpr/primops.cc | 2 +- src/libexpr/primops/fetchTree.cc | 18 ++++++------------ src/libutil/experimental-features.cc | 2 ++ 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index 204daab6e..e6bc7e915 100644 --- a/doc/manual/src/language/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md @@ -18,6 +18,8 @@ These constants are built into the Nix language evaluator: identifier for the Nix installation on which the expression is being evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + - [`builtins.currentTime`]{#builtins-currentTime} (integer) Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. @@ -36,7 +38,6 @@ These constants are built into the Nix language evaluator: 1683705525 ``` - > **Warning** - > - > This is intended for testing and debugging only. - > Do not use it in production systems, since an impure, that is, non-constant value will break reproducibility. + The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation. + + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a90ff34c0..ef8c3e39b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -745,7 +745,13 @@ struct EvalSettings : Config )"}; Setting pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + )" + }; Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6fbd66389..7f36b021d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1502,7 +1502,7 @@ static RegisterPrimOp primop_storePath({ causes the path to be *copied* again to the Nix store, resulting in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). - This function is not available in pure evaluation mode. + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). )", .fun = prim_storePath, }); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index cd7039025..fe880aaa8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -286,9 +286,9 @@ static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", .args = {"url"}, .doc = R"( - Download the specified URL and return the path of the downloaded - file. This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Download the specified URL and return the path of the downloaded file. + + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchurl, }); @@ -338,8 +338,7 @@ static RegisterPrimOp primop_fetchTarball({ stdenv.mkDerivation { … } ``` - This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchTarball, }); @@ -470,14 +469,9 @@ static RegisterPrimOp primop_fetchGit({ } ``` - > **Note** - > - > Nix will refetch the branch in accordance with - > the option `tarball-ttl`. + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. - > **Note** - > - > This behavior is disabled in *Pure evaluation mode*. + This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). - To fetch the content of a checked-out work directory: diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ad0ec0427..9bebf77e5 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -50,6 +50,8 @@ constexpr std::array xpFeatureDetails = {{ or other impure derivations can rely on impure derivations. Finally, an impure derivation cannot also be [content-addressed](#xp-feature-ca-derivations). + + This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). )", }, { From 36b7e30c1184035e72a7f24dc4656099ac6e208e Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 19 May 2023 22:47:40 +0200 Subject: [PATCH 04/30] Make mounting ssl cert file optional --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 36c89bee9..b50ff1f86 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1777,7 +1777,7 @@ void LocalDerivationGoal::runChild() if (pathExists(path)) ss.push_back(path); - dirsInChroot.emplace(settings.caFile, "/etc/ssl/certs/ca-certificates.crt"); + dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } for (auto & i : ss) dirsInChroot.emplace(i, i); From b14fea6fffaafa30faf67d89b6b6a01c14b3ddf3 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 19 May 2023 23:30:35 +0200 Subject: [PATCH 05/30] Shortcircuit for empty caFile --- src/libstore/build/local-derivation-goal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b50ff1f86..05d6685da 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1777,7 +1777,8 @@ void LocalDerivationGoal::runChild() if (pathExists(path)) ss.push_back(path); - dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + if (settings.caFile != "") + dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } for (auto & i : ss) dirsInChroot.emplace(i, i); From 5a98dd0b3911be970dbaae414b150c35241d2523 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 22 May 2023 02:32:09 +0200 Subject: [PATCH 06/30] Add tests for bind mount of SSL certs in sandbox --- tests/linux-sandbox-cert-test.nix | 29 +++++++++++++++++++++++++++++ tests/linux-sandbox.sh | 24 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 tests/linux-sandbox-cert-test.nix diff --git a/tests/linux-sandbox-cert-test.nix b/tests/linux-sandbox-cert-test.nix new file mode 100644 index 000000000..2b86dad2e --- /dev/null +++ b/tests/linux-sandbox-cert-test.nix @@ -0,0 +1,29 @@ +{ fixed-output }: + +with import ./config.nix; + +mkDerivation ({ + name = "ssl-export"; + buildCommand = '' + # Add some indirection, otherwise grepping into the debug output finds the string. + report () { echo CERT_$1_IN_SANDBOX; } + + if [ -f /etc/ssl/certs/ca-certificates.crt ]; then + content=$( $TEST_ROOT/log) + cat $TEST_ROOT/log + grepQuiet "CERT_${1}_IN_SANDBOX" $TEST_ROOT/log +} + +nocert=$TEST_ROOT/no-cert-file.pem +cert=$TEST_ROOT/some-cert-file.pem +echo -n "CERT_CONTENT" > $cert + +# No cert in sandbox when not a fixed-output derivation +testCert missing normal "$cert" + +# No cert in sandbox when ssl-cert-file is empty +testCert missing fixed-output "" + +# No cert in sandbox when ssl-cert-file is a nonexistent file +testCert missing fixed-output "$nocert" + +# Cert in sandbox when ssl-cert-file is set to an existing file +testCert present fixed-output "$cert" From a420ccc6a8d1ceee8a413f96c61af0c968b1c255 Mon Sep 17 00:00:00 2001 From: Peter Becich Date: Mon, 22 May 2023 21:59:44 -0700 Subject: [PATCH 07/30] `nix flake check`: skip derivations for foreign systems (#7759) `nix flake show` now skips derivations for foreign systems: https://github.com/NixOS/nix/pull/6988 This commit borrows from that to implement the same behavior for `nix flake check`. See "nix flake check breaks on IFD in multi-platform flake" https://github.com/NixOS/nix/issues/4265 --- src/nix/flake.cc | 121 ++++++++++++++++++++++++++++-------------- tests/flakes/check.sh | 4 +- 2 files changed, 85 insertions(+), 40 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3db655aeb..1eea52e15 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -259,6 +259,7 @@ struct CmdFlakeInfo : CmdFlakeMetadata struct CmdFlakeCheck : FlakeCommand { bool build = true; + bool checkAllSystems = false; CmdFlakeCheck() { @@ -267,6 +268,11 @@ struct CmdFlakeCheck : FlakeCommand .description = "Do not build checks.", .handler = {&build, false} }); + addFlag({ + .longName = "all-systems", + .description = "Check the outputs for all systems.", + .handler = {&checkAllSystems, true} + }); } std::string description() override @@ -292,6 +298,7 @@ struct CmdFlakeCheck : FlakeCommand lockFlags.applyNixConfig = true; auto flake = lockFlake(); + auto localSystem = std::string(settings.thisSystem.get()); bool hasErrors = false; auto reportError = [&](const Error & e) { @@ -307,6 +314,8 @@ struct CmdFlakeCheck : FlakeCommand } }; + std::set omittedSystems; + // FIXME: rewrite to use EvalCache. auto resolve = [&] (PosIdx p) { @@ -327,6 +336,15 @@ struct CmdFlakeCheck : FlakeCommand reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; + auto checkSystemType = [&](const std::string & system, const PosIdx pos) { + if (!checkAllSystems && system != localSystem) { + omittedSystems.insert(system); + return false; + } else { + return true; + } + }; + auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional { try { auto drvInfo = getDerivation(*state, v, false); @@ -509,16 +527,18 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) { - auto drvPath = checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - if (drvPath && attr_name == settings.thisSystem.get()) { - drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + if (drvPath && attr_name == settings.thisSystem.get()) { + drvPaths.push_back(DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); + } } } } @@ -529,9 +549,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkApp( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -540,11 +562,13 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) - checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } @@ -553,11 +577,13 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) - checkApp( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } @@ -566,9 +592,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkDerivation( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkDerivation( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -577,9 +605,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos) ) { + checkApp( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -587,6 +617,7 @@ struct CmdFlakeCheck : FlakeCommand state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { checkSystemName(state->symbols[attr.name], attr.pos); + checkSystemType(state->symbols[attr.name], attr.pos); // FIXME: do getDerivations? } } @@ -636,9 +667,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkBundler( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkBundler( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -647,12 +680,14 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) { - checkBundler( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - } + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) { + checkBundler( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + } + }; } } @@ -685,7 +720,15 @@ struct CmdFlakeCheck : FlakeCommand } if (hasErrors) throw Error("some errors were encountered during the evaluation"); - } + + if (!omittedSystems.empty()) { + warn( + "The check omitted these incompatible systems: %s\n" + "Use '--all-systems' to check all.", + concatStringsSep(", ", omittedSystems) + ); + }; + }; }; static Strings defaultTemplateAttrPathsPrefixes{"templates."}; diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh index 865ca61b4..34b82c61c 100644 --- a/tests/flakes/check.sh +++ b/tests/flakes/check.sh @@ -72,6 +72,8 @@ cat > $flakeDir/flake.nix <&1 && fail "nix flake check should have failed" || true) +nix flake check $flakeDir + +checkRes=$(nix flake check --all-systems --keep-going $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "packages.system-1.default" echo "$checkRes" | grepQuiet "packages.system-2.default" From 303858afad8f4084c8b15d66ba02ee083452bb73 Mon Sep 17 00:00:00 2001 From: oxalica Date: Wed, 24 May 2023 01:12:02 +0800 Subject: [PATCH 08/30] Fix typo in error message of too long store path --- src/libstore/path.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 46be54281..552e83114 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -9,8 +9,8 @@ static void checkName(std::string_view path, std::string_view name) if (name.empty()) throw BadStorePath("store path '%s' has an empty name", path); if (name.size() > StorePath::MaxPathLen) - throw BadStorePath("store path '%s' has a name longer than '%d characters", - StorePath::MaxPathLen, path); + throw BadStorePath("store path '%s' has a name longer than %d characters", + path, StorePath::MaxPathLen); // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') From d16a1994fb6048d4ea48090c5aabafb7ad89c84f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 25 May 2023 16:38:29 +0200 Subject: [PATCH 09/30] Properly report build errors on chrooted stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When encountering a build error, Nix moves the output paths out of the chroot into their final location (for “easier debugging of build failures”). However this was broken for chroot stores as it was moving it to the _logical_ location, not the _physical_ one. Fix it by moving to the physical (_real_) location. Fix https://github.com/NixOS/nix/issues/8395 --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 05d6685da..b0289ac75 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -357,7 +357,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() for (auto & [_, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); + auto p = worker.store.toRealPath(status.known->path); if (pathExists(chrootRootDir + p)) renameFile((chrootRootDir + p), p); } From 367fcd9eb590d002cbbae23f22b1395b7d3303e6 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Thu, 2 Mar 2023 17:29:36 -0700 Subject: [PATCH 10/30] installer: don't assume GNU diff macOS Ventura ships with it's own version of diff. Try to output a similar diff with Apple diff as with GNU diff, instead of failing Helps https://github.com/NixOS/nix/issues/7286 --- scripts/install-multi-user.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7c66538b0..79deb2819 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -246,8 +246,15 @@ printf -v _OLD_LINE_FMT "%b" $'\033[1;7;31m-'"$ESC ${RED}%L${ESC}" printf -v _NEW_LINE_FMT "%b" $'\033[1;7;32m+'"$ESC ${GREEN}%L${ESC}" _diff() { + # macOS Ventura doesn't ship with GNU diff. Print similar output except + # without +/- markers or dimming + if diff --version | grep -q "Apple diff"; then + printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" + diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" + else # simple colorized diff comatible w/ pre `--color` versions - diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" + diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" + fi } confirm_rm() { From a382919d7dd696c01c0d5abd04222c2821c0a49d Mon Sep 17 00:00:00 2001 From: polykernel <81340136+polykernel@users.noreply.github.com> Date: Thu, 25 May 2023 00:37:00 -0400 Subject: [PATCH 11/30] primops: lazy evaluation of replaceStrings replacements The primop `builtins.replaceStrings` currently always strictly evaluates the replacement strings, however time and space are wasted for their computation if the corresponding pattern do not occur in the input string. This commit makes the evaluation of the replacement strings lazy by deferring their evaluation to when the corresponding pattern are matched and memoize the result for efficient retrieval on subsequent matches. The testcases for replaceStrings was updated to check for lazy evaluation of the replacements. A note was also added in the release notes to document the behavior change. --- doc/manual/src/release-notes/rl-next.md | 1 + src/libexpr/primops.cc | 25 +++++++++++++------------ src/libexpr/tests/error_traces.cc | 2 +- tests/lang/eval-okay-replacestrings.exp | 2 +- tests/lang/eval-okay-replacestrings.nix | 1 + 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index bc0d41bdf..f92625887 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,3 +4,4 @@ The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). +- The function `builtins.replaceStrings` is now lazy in the value of its second argument `to`, that is a replacee in `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cfae1e5f8..0f3b0a9fe 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3910,13 +3910,8 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a for (auto elem : args[0]->listItems()) from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector> to; - to.reserve(args[1]->listSize()); - for (auto elem : args[1]->listItems()) { - NixStringContext ctx; - auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); - to.emplace_back(s, std::move(ctx)); - } + std::unordered_map cache; + auto to = args[1]->listItems(); NixStringContext context; auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); @@ -3927,10 +3922,19 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a bool found = false; auto i = from.begin(); auto j = to.begin(); - for (; i != from.end(); ++i, ++j) + size_t j_index = 0; + for (; i != from.end(); ++i, ++j, ++j_index) if (s.compare(p, i->size(), *i) == 0) { found = true; - res += j->first; + auto v = cache.find(j_index); + if (v == cache.end()) { + NixStringContext ctx; + auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); + v = (cache.emplace(j_index, ts)).first; + for (auto& path : ctx) + context.insert(path); + } + res += v->second; if (i->empty()) { if (p < s.size()) res += s[p]; @@ -3938,9 +3942,6 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a } else { p += i->size(); } - for (auto& path : j->second) - context.insert(path); - j->second.clear(); break; } if (!found) { diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 24e95ac39..285651256 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -171,7 +171,7 @@ namespace nix { hintfmt("value is %s while a string was expected", "an integer"), hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - ASSERT_TRACE2("replaceStrings [ \"old\" ] [ true ] {}", + ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"", TypeError, hintfmt("value is %s while a string was expected", "a Boolean"), hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp index 72e8274d8..eac67c5fe 100644 --- a/tests/lang/eval-okay-replacestrings.exp +++ b/tests/lang/eval-okay-replacestrings.exp @@ -1 +1 @@ -[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] +[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" "fubar" ] diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix index bd8031fc0..a803e6519 100644 --- a/tests/lang/eval-okay-replacestrings.nix +++ b/tests/lang/eval-okay-replacestrings.nix @@ -8,4 +8,5 @@ with builtins; (replaceStrings [""] ["X"] "abc") (replaceStrings [""] ["X"] "") (replaceStrings ["-"] ["_"] "a-b") + (replaceStrings ["oo" "XX"] ["u" (throw "unreachable")] "foobar") ] From be4890747051de0e489d608fdba65489c45d2b02 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 19:23:09 +0100 Subject: [PATCH 12/30] ci: Always run with sandbox, even on Darwin And fix a test failure in the sandbox due to /home existing on Darwin but not being accessible in the sandbox since it's a symlink to /System/Volumes/Data/home, see https://github.com/NixOS/nix/actions/runs/4205378453/jobs/7297384658#step:6:2127: C++ exception with description "error: getting status of /home/schnitzel/darmstadt/pommes: Operation not permitted" thrown in the test body. On Linux this wasn't a problem because there /home doesn't exist in the sandbox --- .github/workflows/ci.yml | 3 +++ src/libutil/tests/tests.cc | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c06c77043..61e7f4560 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,9 @@ jobs: with: fetch-depth: 0 - uses: cachix/install-nix-action@v20 + with: + # The sandbox would otherwise be disabled by default on Darwin + extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 250e83a38..f3c1e8248 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -202,7 +202,7 @@ namespace nix { } TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } /* ---------------------------------------------------------------------------- From b7cde90c6b479562eb9f1d9df399d04cf9c42aad Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Thu, 11 May 2023 13:09:02 +0200 Subject: [PATCH 13/30] Call getDefaultSSLCertFile() only when none is specified This does pathExists on various paths, which crashes on EPERM in the macOS sandbox. --- src/libstore/globals.cc | 2 ++ src/libstore/globals.hh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4c66d08ee..a196c10e6 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -57,6 +57,8 @@ Settings::Settings() auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (sslOverride != "") caFile = sslOverride; + else if (caFile == "") + caFile = getDefaultSSLCertFile(); /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 31dfe5b4e..34b4f24a7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -842,7 +842,7 @@ public: )"}; Setting caFile{ - this, getDefaultSSLCertFile(), "ssl-cert-file", + this, "", "ssl-cert-file", R"( The path of a file containing CA certificates used to authenticate `https://` downloads. Nix by default will use From a6c78ba367725a81aa631a7df2d0840ddd25faf5 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Thu, 11 May 2023 13:44:16 +0200 Subject: [PATCH 14/30] getDefaultNixPath: ignore EPERM --- src/libexpr/eval.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 585670e69..61012f2ab 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2620,12 +2620,17 @@ Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); + try { + if (pathExists(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } } + } catch (SysError & e) { + // swallow EPERM + if (e.errNo != EPERM) throw; } }; From 2c462486fe0c1bcb5b1142507d2875e98b2418df Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 26 May 2023 15:32:28 +0200 Subject: [PATCH 15/30] create pathAccessible, use it to infer default dirs --- src/libexpr/eval.cc | 15 +++++---------- src/libstore/globals.cc | 4 +--- src/libstore/globals.hh | 2 +- src/libutil/util.cc | 11 +++++++++++ src/libutil/util.hh | 8 ++++++++ 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 61012f2ab..71fd6e6e4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2620,17 +2620,12 @@ Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { - try { - if (pathExists(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } + if (pathAccessible(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); } - } catch (SysError & e) { - // swallow EPERM - if (e.errNo != EPERM) throw; } }; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index a196c10e6..32e9a6ea9 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -57,8 +57,6 @@ Settings::Settings() auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (sslOverride != "") caFile = sslOverride; - else if (caFile == "") - caFile = getDefaultSSLCertFile(); /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); @@ -185,7 +183,7 @@ bool Settings::isWSL1() Path Settings::getDefaultSSLCertFile() { for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) return fn; + if (pathAccessible(fn)) return fn; return ""; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 34b4f24a7..31dfe5b4e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -842,7 +842,7 @@ public: )"}; Setting caFile{ - this, "", "ssl-cert-file", + this, getDefaultSSLCertFile(), "ssl-cert-file", R"( The path of a file containing CA certificates used to authenticate `https://` downloads. Nix by default will use diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3a8309149..aa0a154fd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -266,6 +266,17 @@ bool pathExists(const Path & path) return false; } +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + Path readLink(const Path & path) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index a7907cd14..00fcb9b79 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -120,6 +120,14 @@ struct stat lstat(const Path & path); */ bool pathExists(const Path & path); +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + /** * Read the contents (target) of a symbolic link. The result is not * in any way canonicalised. From e4aebccf209c12714f5547ec5acf08b1fd7f51cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 27 May 2023 17:53:30 +0200 Subject: [PATCH 16/30] Restore Nix 2.3 behaviour for {__impure,__contentAddressed} = false Fixes #8405. --- src/libexpr/primops.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cfae1e5f8..341962dc9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1152,16 +1152,14 @@ drvName, Bindings * attrs, Value & v) if (i->value->type() == nNull) continue; } - if (i->name == state.sContentAddressed) { - contentAddressed = state.forceBool(*i->value, noPos, context_below); - if (contentAddressed) - experimentalFeatureSettings.require(Xp::CaDerivations); + if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) { + contentAddressed = true; + experimentalFeatureSettings.require(Xp::CaDerivations); } - else if (i->name == state.sImpure) { - isImpure = state.forceBool(*i->value, noPos, context_below); - if (isImpure) - experimentalFeatureSettings.require(Xp::ImpureDerivations); + else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) { + isImpure = true; + experimentalFeatureSettings.require(Xp::ImpureDerivations); } /* The `args' attribute is special: it supplies the From 6b56376c89a84388da4f25792c4828baee3cc5e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 22:56:41 +0000 Subject: [PATCH 17/30] Bump cachix/install-nix-action from 20 to 21 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 20 to 21. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v20...v21) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61e7f4560..0f1f6d43f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v21 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -61,7 +61,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v21 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -82,7 +82,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v21 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -109,7 +109,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v21 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From 2e01a53a36737bf9fef6b277d3c8e45827a0c716 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 22:56:44 +0000 Subject: [PATCH 18/30] Bump zeebe-io/backport-action from 1.2.0 to 1.3.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.2.0 to 1.3.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.2.0...v1.3.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index b04723b95..37966bab2 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.2.0 + uses: zeebe-io/backport-action@v1.3.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From f3a88857b46199a44c4389f6e39b59fb9bef70d0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 30 May 2023 11:49:59 +0200 Subject: [PATCH 19/30] maintainers/upload-release.pl: Don't update nix-fallback-paths.nix --- maintainers/upload-release.pl | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 77469148a..07b19c13f 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -15,7 +15,6 @@ my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; my $channelsBucketName = "nix-channels"; -my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; @@ -200,26 +199,22 @@ for my $fn (glob "$tmpDir/*") { } } -# Update nix-fallback-paths.nix. +# Print new nix-fallback-paths.nix. if ($isLatest) { - system("cd $nixpkgsDir && git pull") == 0 or die; - sub getStorePath { my ($jobName) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; } - write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", + print STDERR "nixos/modules/installer/tools/nix-fallback-paths.nix:\n" . "{\n" . " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . - "}\n"); - - system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; + "}\n"; } # Update the "latest" symlink. From 7a5731aa0700a1ae7ec3c2d890df235930446b08 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 30 May 2023 22:58:00 +0200 Subject: [PATCH 20/30] link to mentioned builtin --- doc/manual/src/release-notes/rl-next.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index f92625887..27eb69cb0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,4 +4,4 @@ The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). -- The function `builtins.replaceStrings` is now lazy in the value of its second argument `to`, that is a replacee in `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. +- The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. From 9afd525e1847650589a376a5d3185241e4936b5d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 30 May 2023 22:58:15 +0200 Subject: [PATCH 21/30] update documentation according to release notes --- src/libexpr/primops.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c28e7cbc7..544785adc 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3957,7 +3957,11 @@ static RegisterPrimOp primop_replaceStrings({ .args = {"from", "to", "s"}, .doc = R"( Given string *s*, replace every occurrence of the strings in *from* - with the corresponding string in *to*. For example, + with the corresponding string in *to*. + + The argument *to* is lazy, that is, it is only evaluated when its corresponding pattern in *from* is matched in the string *s* + + Example: ```nix builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" From 76287a9207ce9b0fbab7ae49f6fa405f0c1ef9ab Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Mon, 29 May 2023 18:17:29 +0300 Subject: [PATCH 22/30] distributed-builds.md: Clarify warning ssh access requirements --- doc/manual/src/advanced-topics/distributed-builds.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index fefd10100..8f61380e8 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -39,10 +39,11 @@ contains Nix. > **Warning** > > If you are building via the Nix daemon, it is the Nix daemon user -> account (that is, `root`) that should have SSH access to the remote +> account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote > machine. If you can’t or don’t want to configure `root` to be able to -> access to remote machine, you can use a private Nix store instead by -> passing e.g. `--store ~/my-nix`. +> access the remote machine, you can use a private Nix store instead by +> passing e.g. `--store ~/my-nix` when running a nix command from the +> local machine. The list of remote machines can be specified on the command line or in the Nix configuration file. The former is convenient for testing. For From e2f33e0df10062cfc6d918c9431205cb7c66be4e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 31 May 2023 01:32:02 +0200 Subject: [PATCH 23/30] rewrap lines to use one line per sentence this makes future reviews easier as it reduces diff noise --- doc/manual/src/advanced-topics/distributed-builds.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 8f61380e8..73a113d35 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -38,12 +38,9 @@ contains Nix. > **Warning** > -> If you are building via the Nix daemon, it is the Nix daemon user -> account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote -> machine. If you can’t or don’t want to configure `root` to be able to -> access the remote machine, you can use a private Nix store instead by -> passing e.g. `--store ~/my-nix` when running a nix command from the -> local machine. +> If you are building via the Nix daemon, it is the Nix daemon user account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote machine. +> +> If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine. The list of remote machines can be specified on the command line or in the Nix configuration file. The former is convenient for testing. For From afc24e6a66b5fe3d1fdcc118b12dadda33e08e54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 May 2023 12:38:05 +0200 Subject: [PATCH 24/30] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.16.md | 8 ++++++++ doc/manual/src/release-notes/rl-next.md | 5 ----- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.16.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 606aecd8f..3149af880 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -105,6 +105,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.16 (2023-05-21)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) diff --git a/doc/manual/src/release-notes/rl-2.16.md b/doc/manual/src/release-notes/rl-2.16.md new file mode 100644 index 000000000..97b40d0b8 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.16.md @@ -0,0 +1,8 @@ +# Release 2.16 (2023-05-31) + +* Speed-up of downloads from binary caches. + The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). + The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). + The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). + +* The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 27eb69cb0..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,7 +1,2 @@ # Release X.Y (202?-??-??) -- Speed-up of downloads from binary caches. - The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). - The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). - The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). -- The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. From 008f89fa50fdbdd39de77d48e738f144195cc486 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 May 2023 14:05:00 +0200 Subject: [PATCH 25/30] Typo --- doc/manual/src/SUMMARY.md.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3149af880..69c721b57 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -105,7 +105,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) - - [Release 2.16 (2023-05-21)](release-notes/rl-2.16.md) + - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) From ebed259768d481190ccbaa69a80ef68947ef109f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 May 2023 17:09:38 +0200 Subject: [PATCH 26/30] Bump version --- .version | 2 +- maintainers/release-process.md | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.version b/.version index 752490696..d76bd2ba3 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.16.0 +2.17.0 diff --git a/maintainers/release-process.md b/maintainers/release-process.md index ec9e96489..d85266b81 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -119,8 +119,7 @@ release: TODO: This script requires the right AWS credentials. Document. TODO: This script currently requires a - `/home/eelco/Dev/nix-pristine` and - `/home/eelco/Dev/nixpkgs-pristine`. + `/home/eelco/Dev/nix-pristine`. TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/ @@ -141,7 +140,7 @@ release: $ git checkout master $ git pull $ NEW_VERSION=2.13.0 - $ echo -n $NEW_VERSION > .version + $ echo $NEW_VERSION > .version $ git checkout -b bump-$NEW_VERSION $ git commit -a -m 'Bump version' $ git push --set-upstream origin bump-$NEW_VERSION From a15b2c01c0bac226ed34d8ba7e10a563c4a54890 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 2 Jun 2023 15:38:17 +0000 Subject: [PATCH 27/30] Document manual migration for use-xdg-base-directories (#8044) * Document manual migration for use-xdg-base-directories As there's currently no automatic migration for use-xdg-base-directories option, add instructions for manual migration to the option's description. Co-authored-by: Valentin Gagarin --- src/libstore/globals.hh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 31dfe5b4e..07f524858 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -995,6 +995,18 @@ public: | `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` | | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | + + If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option. + If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`. + This can be achieved with the following shell commands: + + ```sh + nix_state_home=${XDG_STATE_HOME-$HOME/.local/state}/nix + mkdir -p $nix_state_home + mv $HOME/.nix-profile $nix_state_home/profile + mv $HOME/.nix-defexpr $nix_state_home/defexpr + mv $HOME/.nix-channels $nix_state_home/channels + ``` )" }; }; From b64450ed4e25ede3ce83d675c44dcf1ba524790a Mon Sep 17 00:00:00 2001 From: figsoda Date: Sun, 4 Jun 2023 14:50:47 -0400 Subject: [PATCH 28/30] Fix code block formatting in man page --- src/nix/nix.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/nix.md b/src/nix/nix.md index 8a850ae83..6d9e40dbc 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -102,6 +102,7 @@ way: available in the flake. If this is undesirable, specify `path:` explicitly; For example, if `/foo/bar` is a git repository with the following structure: + ``` . └── baz From f5c6b299407e2ad5d05670bf4e89e18ebb113070 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Jun 2023 15:43:12 +0200 Subject: [PATCH 29/30] Fix SourcePath::resolveSymlinks() This fixes handling of symlinks that start with '..', and symlink targets that contain symlinks themselves. --- src/libfetchers/input-accessor.cc | 30 ++++++++++++++++++------------ tests/eval.sh | 6 ++++++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index f9909c218..f37a8058b 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -75,22 +75,28 @@ SourcePath SourcePath::resolveSymlinks() const int linksAllowed = 1024; - for (auto & component : path) { - res.path.push(component); - while (true) { - if (auto st = res.maybeLstat()) { + std::list todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.path.pop(); + else { + res.path.push(c); + if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { if (!linksAllowed--) throw Error("infinite symlink recursion in path '%s'", path); - if (st->type != InputAccessor::tSymlink) break; auto target = res.readLink(); + res.path.pop(); if (hasPrefix(target, "/")) - res = CanonPath(target); - else { - res.path.pop(); - res.path.extend(CanonPath(target)); - } - } else - break; + res.path = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } } } diff --git a/tests/eval.sh b/tests/eval.sh index 066d8fc36..b81bb1e2c 100644 --- a/tests/eval.sh +++ b/tests/eval.sh @@ -35,3 +35,9 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true' # Check that symlink cycles don't cause a hang. ln -sfn cycle.nix $TEST_ROOT/cycle.nix (! nix eval --file $TEST_ROOT/cycle.nix) + +# Check that relative symlinks are resolved correctly. +mkdir -p $TEST_ROOT/xyzzy $TEST_ROOT/foo +ln -sfn ../xyzzy $TEST_ROOT/foo/bar +printf 123 > $TEST_ROOT/xyzzy/default.nix +[[ $(nix eval --impure --expr "import $TEST_ROOT/foo/bar") = 123 ]] From c47f744e05d8c80bd65083b0fc0fbfef2ff0c08f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 1 Jun 2023 14:14:13 +0100 Subject: [PATCH 30/30] Also skip makeStoreWritable when read-only=true. --- src/libstore/local-store.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4188d413f..17385cdfb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -190,7 +190,11 @@ LocalStore::LocalStore(const Params & params) /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); - makeStoreWritable(); + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } else { + makeStoreWritable(); + } createDirs(linksDir); Path profilesDir = stateDir + "/profiles"; createDirs(profilesDir); @@ -202,10 +206,6 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } - if (readOnly) { - experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); - } - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); if (!readOnly) {