Merge pull request #9061 from edolstra/stabilize-fetchTree

fetchTree cleanup

src/libexpr/primops/fetchTree.cc

@@ -25,7 +25,6 @@ void emitTreeAttrs(
 
     auto attrs = state.buildBindings(10);
 
-
     state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath));
 
     // FIXME: support arbitrary input attributes.
@@ -71,36 +70,10 @@ void emitTreeAttrs(
     v.mkAttrs(attrs);
 }
 
-std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
-{
-    state.checkURI(uri);
-    if (uri.find("://") == std::string::npos) {
-        const auto p = ParsedURL {
-            .scheme = defaultScheme,
-            .authority = "",
-            .path = uri
-        };
-        return p.to_string();
-    } else {
-        return uri;
-    }
-}
-
-std::string fixURIForGit(std::string uri, EvalState & state)
-{
-    /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
-     * them by removing the `:` and assuming a scheme of `ssh://`
-     * */
-    static std::regex scp_uri("([^/]*)@(.*):(.*)");
-    if (uri[0] != '/' && std::regex_match(uri, scp_uri))
-        return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
-    else
-        return fixURI(uri, state);
-}
-
 struct FetchTreeParams {
     bool emptyRevFallback = false;
     bool allowNameArgument = false;
+    bool isFetchGit = false;
 };
 
 static void fetchTree(
@@ -108,11 +81,12 @@ static void fetchTree(
     const PosIdx pos,
     Value * * args,
     Value & v,
-    std::optional<std::string> type,
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
     NixStringContext context;
+    std::optional<std::string> type;
+    if (params.isFetchGit) type = "git";
 
     state.forceValue(*args[0], pos);
 
@@ -142,10 +116,8 @@ static void fetchTree(
             if (attr.value->type() == nPath || attr.value->type() == nString) {
                 auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned();
                 attrs.emplace(state.symbols[attr.name],
-                    state.symbols[attr.name] == "url"
-                    ? type == "git"
-                      ? fixURIForGit(s, state)
-                      : fixURI(s, state)
+                    params.isFetchGit && state.symbols[attr.name] == "url"
+                    ? fixGitURL(s)
                     : s);
             }
             else if (attr.value->type() == nBool)
@@ -170,22 +142,24 @@ static void fetchTree(
                 "while evaluating the first argument passed to the fetcher",
                 false, false).toOwned();
 
-        if (type == "git") {
+        if (params.isFetchGit) {
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
-            attrs.emplace("url", fixURIForGit(url, state));
+            attrs.emplace("url", fixGitURL(url));
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
-            input = fetchers::Input::fromURL(fixURI(url, state));
+            input = fetchers::Input::fromURL(url);
         }
     }
 
-    if (!evalSettings.pureEval && !input.isDirect())
+    if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
         input = lookupInRegistries(state.store, input).first;
 
     if (evalSettings.pureEval && !input.isLocked())
         state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
 
+    state.checkURI(input.toURLString());
+
     auto [tree, input2] = input.fetch(state.store);
 
     state.allowPath(tree.storePath);
@@ -195,7 +169,7 @@ static void fetchTree(
 
 static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
+    fetchTree(state, pos, args, v, { });
 }
 
 static RegisterPrimOp primop_fetchTree({
@@ -203,12 +177,12 @@ static RegisterPrimOp primop_fetchTree({
     .args = {"input"},
     .doc = R"(
       Fetch a source tree or a plain file using one of the supported backends.
-      *input* can be an attribute set representation of [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) or a URL.
-      The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is allowed.
+      *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax.
+      The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled.
 
       Here are some examples of how to use `fetchTree`:
 
-      - Fetch a GitHub repository:
+      - Fetch a GitHub repository using the attribute set representation:
 
           ```nix
           builtins.fetchTree {
@@ -219,7 +193,7 @@ static RegisterPrimOp primop_fetchTree({
           }
           ```
 
-        This evaluates to attribute set:
+        This evaluates to the following attribute set:
 
           ```
           {
@@ -231,10 +205,11 @@ static RegisterPrimOp primop_fetchTree({
             shortRev = "ae2e6b3";
           }
           ```
-      - Fetch a single file from a URL:
 
-          ```nix
-          builtins.fetchTree "https://example.com/"
+      - Fetch the same GitHub repository using the URL-like syntax:
+
+          ```
+          builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
           ```
     )",
     .fun = prim_fetchTree,
@@ -388,7 +363,12 @@ static RegisterPrimOp primop_fetchTarball({
 
 static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
+    fetchTree(state, pos, args, v,
+        FetchTreeParams {
+            .emptyRevFallback = true,
+            .allowNameArgument = true,
+            .isFetchGit = true
+        });
 }
 
 static RegisterPrimOp primop_fetchGit({

src/libfetchers/git.cc

@@ -293,7 +293,6 @@ struct GitInputScheme : InputScheme
             if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
                 throw Error("unsupported Git input attribute '%s'", name);
 
-        parseURL(getStrAttr(attrs, "url"));
         maybeGetBoolAttr(attrs, "shallow");
         maybeGetBoolAttr(attrs, "submodules");
         maybeGetBoolAttr(attrs, "allRefs");
@@ -305,6 +304,9 @@ struct GitInputScheme : InputScheme
 
         Input input;
         input.attrs = attrs;
+        auto url = fixGitURL(getStrAttr(attrs, "url"));
+        parseURL(url);
+        input.attrs["url"] = url;
         return input;
     }
 

src/libutil/url.cc

@@ -158,4 +158,21 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme)
     };
 }
 
+std::string fixGitURL(const std::string & url)
+{
+    std::regex scpRegex("([^/]*)@(.*):(.*)");
+    if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex))
+        return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3");
+    else {
+        if (url.find("://") == std::string::npos) {
+            return (ParsedURL {
+                .scheme = "file",
+                .authority = "",
+                .path = url
+            }).to_string();
+        } else
+            return url;
+    }
+}
+
 }

src/libutil/url.hh

@@ -45,4 +45,9 @@ struct ParsedUrlScheme {
 
 ParsedUrlScheme parseUrlScheme(std::string_view scheme);
 
+/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
+   them by removing the `:` and assuming a scheme of `ssh://`. Also
+   changes absolute paths into file:// URLs. */
+std::string fixGitURL(const std::string & url);
+
 }

src/nix/flake.md

@@ -182,6 +182,12 @@ Currently the `type` attribute can be one of the following:
   git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)?
   ```
 
+  or
+
+  ```
+  <user>@<server>:<path>
+  ```
+
   The `ref` attribute defaults to resolving the `HEAD` reference.
 
   The `rev` attribute must denote a commit that exists in the branch

tests/functional/fetchGit.sh

@@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP
 path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
 path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
 [[ $path0 = $path0_ ]]
+path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath")
+[[ $path0 = $path0_ ]]
 export _NIX_FORCE_HTTP=1
 [[ $(tail -n 1 $path0/hello) = "hello" ]]
 

tests/nixos/github-flakes.nix

@@ -186,6 +186,10 @@ in
     client.succeed("nix registry pin nixpkgs")
     client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2")
 
+    # Test fetchTree on a github URL.
+    hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'")
+    assert hash == info['locked']['narHash']
+
     # Shut down the web server. The flake should be cached on the client.
     github.succeed("systemctl stop httpd.service")