Merge 78b5b4c105 into 1cd48008f0

Fixes flake-regressions/tests/DeterminateSystems/fh/0.1.10:

  error: fetching final input '{"final":true,"narHash":"sha256-0dZpggYjjmWEk+rGixiBHOHuQfLzEzNfrtjSig04s6Q=","rev":"9ccae1754eec0341b640d5705302ac0923d22875","revCount":1618,"type":"tarball","url":"https://api.flakehub.com/f/pinned/nix-community/fenix/0.1.1618%2Brev-9ccae1754eec0341b640d5705302ac0923d22875/018aea4c-03c9-7734-95d5-b84cc8881e3d/source.tar.gz"}' resulted in different input '{"final":true,"lastModified":1696141234,"narHash":"sha256-0dZpggYjjmWEk+rGixiBHOHuQfLzEzNfrtjSig04s6Q=","rev":"9ccae1754eec0341b640d5705302ac0923d22875","revCount":1618,"type":"tarball","url":"https://api.flakehub.com/f/pinned/nix-community/fenix/0.1.1618%2Brev-9ccae1754eec0341b640d5705302ac0923d22875/018aea4c-03c9-7734-95d5-b84cc8881e3d/source.tar.gz"}'

maintainers/README.md

@@ -29,11 +29,7 @@ We aim to achieve this by improving the contributor experience and attracting mo
 
 ## Members
 
-- Eelco Dolstra (@edolstra) – Team lead
-- Valentin Gagarin (@fricklerhandwerk)
-- Thomas Bereknyei (@tomberek)
-- Robert Hensing (@roberth)
-- John Ericson (@Ericson2314)
+See https://nixos.org/community/teams/nix/ for the current team membership.
 
 The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team).
 

maintainers/onboarding.md

@@ -0,0 +1,6 @@
+
+# Onboarding a new team member
+
+- https://github.com/NixOS/nixos-homepage/
+- https://github.com/orgs/NixOS/teams/nix-team
+- Matrix room

src/libexpr/call-flake.nix

@@ -44,7 +44,8 @@ let
               overrides.${key}.sourceInfo
             else
               # FIXME: remove obsolete node.info.
-              fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
+              # Note: lock file entries are always final.
+              fetchTree (node.info or {} // removeAttrs node.locked ["dir"] // { final = true; });
 
           subdir = overrides.${key}.dir or node.locked.dir or "";
 

src/libfetchers/fetchers.cc

@@ -3,6 +3,7 @@
 #include "source-path.hh"
 #include "fetch-to-store.hh"
 #include "json-utils.hh"
+#include "store-path-accessor.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -100,7 +101,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
     auto allowedAttrs = inputScheme->allowedAttrs();
 
     for (auto & [name, _] : attrs)
-        if (name != "type" && allowedAttrs.count(name) == 0)
+        if (name != "type" && name != "final" && allowedAttrs.count(name) == 0)
             throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
 
     auto res = inputScheme->inputFromAttrs(settings, attrs);
@@ -145,6 +146,11 @@ bool Input::isLocked() const
     return scheme && scheme->isLocked(*this);
 }
 
+bool Input::isFinal() const
+{
+    return maybeGetBoolAttr(attrs, "final").value_or(false);
+}
+
 Attrs Input::toAttrs() const
 {
     return attrs;
@@ -179,6 +185,14 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
             auto narHash = store->queryPathInfo(storePath)->narHash;
             final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
+            // FIXME: we would like to mark inputs as final in
+            // getAccessorUnchecked(), but then we can't add
+            // narHash. Or maybe narHash should be excluded from the
+            // concept of "final" inputs?
+            final.attrs.insert_or_assign("final", Explicit<bool>(true));
+
+            assert(final.isFinal());
+
             scheme->checkLocks(*this, final);
 
             return {storePath, final};
@@ -221,6 +235,10 @@ void InputScheme::checkLocks(const Input & specified, const Input & final) const
             throw Error("'revCount' attribute mismatch in input '%s', expected %d",
                 final.to_string(), *prevRevCount);
     }
+
+    if (specified.isFinal() && specified.attrs != final.attrs)
+        throw Error("fetching final input '%s' resulted in different input '%s'",
+            attrsToJSON(specified.attrs), attrsToJSON(final.attrs));
 }
 
 std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
@@ -244,6 +262,36 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
     if (!scheme)
         throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
 
+    /* The tree may already be in the Nix store, or it could be
+       substituted (which is often faster than fetching from the
+       original source). So check that. We only do this for final
+       inputs, otherwise there is a risk that we don't return the
+       same attributes (like `lastModified`) that the "real" fetcher
+       would return.
+
+       FIXME: add a setting to disable this.
+       FIXME: substituting may be slower than fetching normally,
+       e.g. for fetchers like Git that are incremental!
+    */
+    if (isFinal() && getNarHash()) {
+        try {
+            auto storePath = computeStorePath(*store);
+
+            store->ensurePath(storePath);
+
+            debug("using substituted/cached input '%s' in '%s'",
+                to_string(), store->printStorePath(storePath));
+
+            auto accessor = makeStorePathAccessor(store, storePath);
+
+            accessor->fingerprint = scheme->getFingerprint(store, *this);
+
+            return {accessor, *this};
+        } catch (Error & e) {
+            debug("substitution of input '%s' failed: %s", to_string(), e.what());
+        }
+    }
+
     auto [accessor, final] = scheme->getAccessor(store, *this);
 
     assert(!accessor->fingerprint);

src/libfetchers/fetchers.hh

@@ -84,11 +84,21 @@ public:
     bool isDirect() const;
 
     /**
-     * Check whether this is a "locked" input, that is,
-     * one that contains a commit hash or content hash.
+     * Check whether this is a "locked" input, that is, it has
+     * attributes like a Git revision or NAR hash that uniquely
+     * identify its contents.
      */
     bool isLocked() const;
 
+    /**
+     * Check whether this is a "final" input, meaning that fetching it
+     * will not add or change any attributes. For instance, a Git
+     * input with a `rev` attribute but without a `lastModified`
+     * attribute is considered locked but not final. Only "final"
+     * inputs can be substituted from a binary cache.
+     */
+    bool isFinal() const;
+
     bool operator ==(const Input & other) const noexcept;
 
     bool contains(const Input & other) const;
@@ -144,6 +154,10 @@ public:
     /**
      * For locked inputs, return a string that uniquely specifies the
      * content of the input (typically a commit hash or content hash).
+     *
+     * Only known-equivalent inputs should return the same fingerprint.
+     *
+     * This is not a stable identifier between Nix versions, but not guaranteed to change either.
      */
     std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
@@ -212,24 +226,15 @@ struct InputScheme
      */
     virtual std::optional<ExperimentalFeature> experimentalFeature() const;
 
+    /// See `Input::isDirect()`.
     virtual bool isDirect(const Input & input) const
     { return true; }
 
-    /**
-     * A sufficiently unique string that can be used as a cache key to identify the `input`.
-     *
-     * Only known-equivalent inputs should return the same fingerprint.
-     *
-     * This is not a stable identifier between Nix versions, but not guaranteed to change either.
-     */
+    /// See `Input::getFingerprint()`.
     virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     { return std::nullopt; }
 
-    /**
-     * Return `true` if this input is considered "locked", i.e. it has
-     * attributes like a Git revision or NAR hash that uniquely
-     * identify its contents.
-     */
+    /// See `Input::isLocked()`.
     virtual bool isLocked(const Input & input) const
     { return false; }
 

src/libfetchers/path.cc

@@ -72,6 +72,7 @@ struct PathInputScheme : InputScheme
         auto query = attrsToQuery(input.attrs);
         query.erase("path");
         query.erase("type");
+        query.erase("final");
         return ParsedURL {
             .scheme = "path",
             .path = getStrAttr(input.attrs, "path"),

src/libfetchers/tarball.cc

@@ -384,7 +384,11 @@ struct TarballInputScheme : CurlInputScheme
             input = immutableInput;
         }
 
-        if (result.lastModified && !input.attrs.contains("lastModified"))
+        /* If we got a lastModified and the input is not final and
+           doesn't have one, then return it. Note that we don't do
+           this if the input is final for compatibility with old lock
+           files that didn't include lastModified. */
+        if (result.lastModified && !_input.isFinal() && !input.attrs.contains("lastModified"))
             input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         input.attrs.insert_or_assign("narHash",

src/libflake/flake/flake.cc

@@ -85,7 +85,6 @@ static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos
         state.forceValue(value, pos);
 }
 
-
 static void expectType(EvalState & state, ValueType type,
     Value & value, const PosIdx pos)
 {

src/libflake/flake/lockfile.cc

@@ -46,6 +46,10 @@ LockedNode::LockedNode(
     if (!lockedRef.input.isLocked())
         throw Error("lock file contains unlocked input '%s'",
             fetchers::attrsToJSON(lockedRef.input.toAttrs()));
+
+    // For backward compatibility, lock file entries are implicitly final.
+    assert(!lockedRef.input.attrs.contains("final"));
+    lockedRef.input.attrs.insert_or_assign("final", Explicit<bool>(true));
 }
 
 StorePath LockedNode::computeStorePath(Store & store) const
@@ -53,7 +57,6 @@ StorePath LockedNode::computeStorePath(Store & store) const
     return lockedRef.input.computeStorePath(store);
 }
 
-
 static std::shared_ptr<Node> doFind(const ref<Node> & root, const InputPath & path, std::vector<InputPath> & visited)
 {
     auto pos = root;
@@ -191,6 +194,11 @@ std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
         if (auto lockedNode = node.dynamic_pointer_cast<const LockedNode>()) {
             n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
             n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
+            /* For backward compatibility, omit the "final"
+               attribute. We never allow non-final inputs in lock files
+               anyway. */
+            assert(lockedNode->lockedRef.input.isFinal());
+            n["locked"].erase("final");
             if (!lockedNode->isFlake)
                 n["flake"] = false;
         }
@@ -239,7 +247,7 @@ std::optional<FlakeRef> LockFile::isUnlocked() const
     for (auto & i : nodes) {
         if (i == ref<const Node>(root)) continue;
         auto node = i.dynamic_pointer_cast<const LockedNode>();
-        if (node && !node->lockedRef.input.isLocked())
+        if (node && (!node->lockedRef.input.isLocked() || !node->lockedRef.input.isFinal()))
             return node->lockedRef;
     }
 

src/libflake/flake/lockfile.hh

@@ -68,8 +68,8 @@ struct LockFile
     std::pair<std::string, KeyMap> to_string() const;
 
     /**
-     * Check whether this lock file has any unlocked inputs. If so,
-     * return one.
+     * Check whether this lock file has any unlocked or non-final
+     * inputs. If so, return one.
      */
     std::optional<FlakeRef> isUnlocked() const;