From 7455cbfda0622fd20577a7e8473e1f26f413bd54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2024 16:54:31 +0200 Subject: [PATCH 1/2] Fetch Git submodules lazily This isn't particularly useful right now, since we still copy the entire Git input to the store. But once we have lazy trees, this will avoid fetching submodules unless they're actually needed. --- src/libfetchers/git.cc | 38 +++++++++------- src/libutil/lazy-source-accessor.hh | 70 +++++++++++++++++++++++++++++ src/libutil/meson.build | 1 + 3 files changed, 93 insertions(+), 16 deletions(-) create mode 100644 src/libutil/lazy-source-accessor.hh diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 076c757c5..4b11bde2c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -10,6 +10,7 @@ #include "processes.hh" #include "git.hh" #include "mounted-source-accessor.hh" +#include "lazy-source-accessor.hh" #include "git-utils.hh" #include "logging.hh" #include "finally.hh" @@ -645,22 +646,27 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) { - auto resolved = repo->resolveSubmoduleUrl(submodule.url); - debug("Git submodule %s: %s %s %s -> %s", - submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", resolved); - if (submodule.branch != "") - attrs.insert_or_assign("ref", submodule.branch); - attrs.insert_or_assign("rev", submoduleRev.gitRev()); - attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); - attrs.insert_or_assign("submodules", Explicit{ true }); - attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); - auto [submoduleAccessor, submoduleInput2] = - submoduleInput.getAccessor(store); - submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); + auto submoduleAccessor = make_ref([repoDir, submodule, submoduleRev, exportIgnore, store, settings(input.settings)]() + { + auto repo = GitRepo::openRepo(repoDir); + auto resolved = repo->resolveSubmoduleUrl(submodule.url); + debug("Git submodule %s: %s %s %s -> %s", + submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") + attrs.insert_or_assign("ref", submodule.branch); + attrs.insert_or_assign("rev", submoduleRev.gitRev()); + attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); + attrs.insert_or_assign("submodules", Explicit{ true }); + attrs.insert_or_assign("allRefs", Explicit{ true }); + auto submoduleInput = fetchers::Input::fromAttrs(*settings, std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.getAccessor(store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); + return submoduleAccessor; + }); mounts.insert_or_assign(submodule.path, submoduleAccessor); } diff --git a/src/libutil/lazy-source-accessor.hh b/src/libutil/lazy-source-accessor.hh new file mode 100644 index 000000000..8758a7443 --- /dev/null +++ b/src/libutil/lazy-source-accessor.hh @@ -0,0 +1,70 @@ +#pragma once + +#include "source-accessor.hh" +#include "sync.hh" + +#include + +namespace nix { + +/** + * A wrapper `SourceAccessor` that lazily constructs an underlying + * `SourceAccessor`. + */ +struct LazySourceAccessor : SourceAccessor +{ + using Fun = std::function()>; + + Sync, Fun>> next_; + + LazySourceAccessor(Fun next) + : next_{{std::move(next)}} + { + } + + ref getNext() + { + auto next(next_.lock()); + if (auto accessor = std::get_if>(&*next)) + return *accessor; + else { + auto fun = std::get(*next); + auto acc = fun(); + *next = acc; + return acc; + } + abort(); + } + + std::string readFile(const CanonPath & path) override + { + return getNext()->readFile(path); + } + + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + { + return getNext()->readFile(path, sink, sizeCallback); + } + + bool pathExists(const CanonPath & path) override + { + return getNext()->pathExists(path); + } + + std::optional maybeLstat(const CanonPath & path) override + { + return getNext()->maybeLstat(path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return getNext()->readDirectory(path); + } + + std::string readLink(const CanonPath & path) override + { + return getNext()->readLink(path); + } +}; + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 797dcae6d..40cfada1a 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -202,6 +202,7 @@ headers = [config_h] + files( 'hilite.hh', 'json-impls.hh', 'json-utils.hh', + 'lazy-source-accessor.hh', 'logging.hh', 'lru-cache.hh', 'memory-source-accessor.hh', From 353b2298b5605b469ec0843b9d6ab55a22260987 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2024 18:49:16 +0200 Subject: [PATCH 2/2] Remove unreachable code --- src/libutil/lazy-source-accessor.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/lazy-source-accessor.hh b/src/libutil/lazy-source-accessor.hh index 8758a7443..544407b63 100644 --- a/src/libutil/lazy-source-accessor.hh +++ b/src/libutil/lazy-source-accessor.hh @@ -33,7 +33,6 @@ struct LazySourceAccessor : SourceAccessor *next = acc; return acc; } - abort(); } std::string readFile(const CanonPath & path) override