Merge pull request #8397 from NixLayeredStore/overlayfs-store

Local Overlay Store

Makefile

@@ -48,6 +48,7 @@ makefiles += \
   tests/functional/ca/local.mk \
   tests/functional/git-hashing/local.mk \
   tests/functional/dyn-drv/local.mk \
+  tests/functional/local-overlay-store/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
 endif

src/libstore/gc.cc

@@ -665,7 +665,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         results.paths.insert(path);
 
         uint64_t bytesFreed;
-        deletePath(realPath, bytesFreed);
+        deleteStorePath(realPath, bytesFreed);
+
         results.bytesFreed += bytesFreed;
 
         if (results.bytesFreed > options.maxFreed) {
@@ -752,7 +753,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 auto i = referrersCache.find(*path);
                 if (i == referrersCache.end()) {
                     StorePathSet referrers;
-                    queryReferrers(*path, referrers);
+                    queryGCReferrers(*path, referrers);
                     referrersCache.emplace(*path, std::move(referrers));
                     i = referrersCache.find(*path);
                 }
@@ -879,7 +880,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
             if (unlink(path.c_str()) == -1)
                 throw SysError("deleting '%1%'", path);
 
-            /* Do not accound for deleted file here. Rely on deletePath()
+            /* Do not account for deleted file here. Rely on deletePath()
                accounting.  */
         }
 

src/libstore/local-overlay-store.cc

@@ -0,0 +1,292 @@
+#include "local-overlay-store.hh"
+#include "callback.hh"
+#include "realisation.hh"
+#include "processes.hh"
+#include "url.hh"
+#include <regex>
+
+namespace nix {
+
+std::string LocalOverlayStoreConfig::doc()
+{
+    return
+        #include "local-overlay-store.md"
+        ;
+}
+
+Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) {
+    return upperLayer + "/" + path.to_string();
+}
+
+LocalOverlayStore::LocalOverlayStore(const Params & params)
+    : StoreConfig(params)
+    , LocalFSStoreConfig(params)
+    , LocalStoreConfig(params)
+    , LocalOverlayStoreConfig(params)
+    , Store(params)
+    , LocalFSStore(params)
+    , LocalStore(params)
+    , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast<LocalFSStore>())
+{
+    if (checkMount.get()) {
+        std::smatch match;
+        std::string mountInfo;
+        auto mounts = readFile("/proc/self/mounts");
+        auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))");
+
+        // Mount points can be stacked, so there might be multiple matching entries.
+        // Loop until the last match, which will be the current state of the mount point.
+        while (std::regex_search(mounts, match, regex)) {
+            mountInfo = match.str();
+            mounts = match.suffix();
+        }
+
+        auto checkOption = [&](std::string option, std::string value) {
+            return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)"));
+        };
+
+        auto expectedLowerDir = lowerStore->realStoreDir.get();
+        if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) {
+            debug("expected lowerdir: %s", expectedLowerDir);
+            debug("expected upperdir: %s", upperLayer);
+            debug("actual mount: %s", mountInfo);
+            throw Error("overlay filesystem '%s' mounted incorrectly",
+                realStoreDir.get());
+        }
+    }
+}
+
+
+void LocalOverlayStore::registerDrvOutput(const Realisation & info)
+{
+    // First do queryRealisation on lower layer to populate DB
+    auto res = lowerStore->queryRealisation(info.id);
+    if (res)
+        LocalStore::registerDrvOutput(*res);
+
+    LocalStore::registerDrvOutput(info);
+}
+
+
+void LocalOverlayStore::queryPathInfoUncached(const StorePath & path,
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
+{
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
+    LocalStore::queryPathInfoUncached(path,
+        {[this, path, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+            try {
+                auto info = fut.get();
+                if (info)
+                    return (*callbackPtr)(std::move(info));
+            } catch (...) {
+                return callbackPtr->rethrow();
+            }
+            // If we don't have it, check lower store
+            lowerStore->queryPathInfo(path,
+                {[path, callbackPtr](std::future<ref<const ValidPathInfo>> fut) {
+                    try {
+                        (*callbackPtr)(fut.get().get_ptr());
+                    } catch (...) {
+                        return callbackPtr->rethrow();
+                    }
+                }});
+        }});
+}
+
+
+void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput,
+    Callback<std::shared_ptr<const Realisation>> callback) noexcept
+{
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
+    LocalStore::queryRealisationUncached(drvOutput,
+        {[this, drvOutput, callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) {
+            try {
+                auto info = fut.get();
+                if (info)
+                    return (*callbackPtr)(std::move(info));
+            } catch (...) {
+                return callbackPtr->rethrow();
+            }
+            // If we don't have it, check lower store
+            lowerStore->queryRealisation(drvOutput,
+                {[callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) {
+                    try {
+                        (*callbackPtr)(fut.get());
+                    } catch (...) {
+                        return callbackPtr->rethrow();
+                    }
+                }});
+        }});
+}
+
+
+bool LocalOverlayStore::isValidPathUncached(const StorePath & path)
+{
+    auto res = LocalStore::isValidPathUncached(path);
+    if (res) return res;
+    res = lowerStore->isValidPath(path);
+    if (res) {
+        // Get path info from lower store so upper DB genuinely has it.
+        auto p = lowerStore->queryPathInfo(path);
+        // recur on references, syncing entire closure.
+        for (auto & r : p->references)
+            if (r != path)
+                isValidPath(r);
+        LocalStore::registerValidPath(*p);
+    }
+    return res;
+}
+
+
+void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & referrers)
+{
+    LocalStore::queryReferrers(path, referrers);
+    lowerStore->queryReferrers(path, referrers);
+}
+
+
+void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers)
+{
+    LocalStore::queryReferrers(path, referrers);
+}
+
+
+StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path)
+{
+    auto res = LocalStore::queryValidDerivers(path);
+    for (auto p : lowerStore->queryValidDerivers(path))
+        res.insert(p);
+    return res;
+}
+
+
+std::optional<StorePath> LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart)
+{
+    auto res = LocalStore::queryPathFromHashPart(hashPart);
+    if (res)
+        return res;
+    else
+        return lowerStore->queryPathFromHashPart(hashPart);
+}
+
+
+void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos)
+{
+    // First, get any from lower store so we merge
+    {
+        StorePathSet notInUpper;
+        for (auto & [p, _] : infos)
+            if (!LocalStore::isValidPathUncached(p)) // avoid divergence
+                notInUpper.insert(p);
+        auto pathsInLower = lowerStore->queryValidPaths(notInUpper);
+        ValidPathInfos inLower;
+        for (auto & p : pathsInLower)
+            inLower.insert_or_assign(p, *lowerStore->queryPathInfo(p));
+        LocalStore::registerValidPaths(inLower);
+    }
+    // Then do original request
+    LocalStore::registerValidPaths(infos);
+}
+
+
+void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
+    LocalStore::collectGarbage(options, results);
+
+    remountIfNecessary();
+}
+
+
+void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed)
+{
+    auto mergedDir = realStoreDir.get() + "/";
+    if (path.substr(0, mergedDir.length()) != mergedDir) {
+        warn("local-overlay: unexpected gc path '%s' ", path);
+        return;
+    }
+
+    StorePath storePath = {path.substr(mergedDir.length())};
+    auto upperPath = toUpperPath(storePath);
+
+    if (pathExists(upperPath)) {
+        debug("upper exists: %s", path);
+        if (lowerStore->isValidPath(storePath)) {
+            debug("lower exists: %s", storePath.to_string());
+            // Path also exists in lower store.
+            // We must delete via upper layer to avoid creating a whiteout.
+            deletePath(upperPath, bytesFreed);
+            _remountRequired = true;
+        } else {
+            // Path does not exist in lower store.
+            // So we can delete via overlayfs and not need to remount.
+            LocalStore::deleteStorePath(path, bytesFreed);
+        }
+    }
+}
+
+
+void LocalOverlayStore::optimiseStore()
+{
+    Activity act(*logger, actOptimiseStore);
+
+    // Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer
+    auto paths = queryAllValidPaths();
+
+    act.progress(0, paths.size());
+
+    uint64_t done = 0;
+
+    for (auto & path : paths) {
+        if (lowerStore->isValidPath(path)) {
+            uint64_t bytesFreed = 0;
+            // Deduplicate store path
+            deleteStorePath(Store::toRealPath(path), bytesFreed);
+        }
+        done++;
+        act.progress(done, paths.size());
+    }
+
+    remountIfNecessary();
+}
+
+
+LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag repair)
+{
+    StorePathSet done;
+
+    auto existsInStoreDir = [&](const StorePath & storePath) {
+        return pathExists(realStoreDir.get() + "/" + storePath.to_string());
+    };
+
+    bool errors = false;
+    StorePathSet validPaths;
+
+    for (auto & i : queryAllValidPaths())
+        verifyPath(i, existsInStoreDir, done, validPaths, repair, errors);
+
+    return {
+        .errors = errors,
+        .validPaths = validPaths,
+    };
+}
+
+
+void LocalOverlayStore::remountIfNecessary()
+{
+    if (!_remountRequired) return;
+
+    if (remountHook.get().empty()) {
+        warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get());
+    } else {
+        runProgram(remountHook, false, {realStoreDir});
+    }
+
+    _remountRequired = false;
+}
+
+
+static RegisterStoreImplementation<LocalOverlayStore, LocalOverlayStoreConfig> regLocalOverlayStore;
+
+}

src/libstore/local-overlay-store.hh

@@ -0,0 +1,215 @@
+#include "local-store.hh"
+
+namespace nix {
+
+/**
+ * Configuration for `LocalOverlayStore`.
+ */
+struct LocalOverlayStoreConfig : virtual LocalStoreConfig
+{
+    LocalOverlayStoreConfig(const StringMap & params)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , LocalStoreConfig(params)
+    { }
+
+    const Setting<std::string> lowerStoreUri{(StoreConfig*) this, "", "lower-store",
+        R"(
+          [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
+
+          Must be a store with a store dir on the file system.
+          Must be used as OverlayFS lower layer for this store's store dir.
+        )"};
+
+    const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer",
+        R"(
+          Directory containing the OverlayFS upper layer for this store's store dir.
+        )"};
+
+    Setting<bool> checkMount{(StoreConfig*) this, true, "check-mount",
+        R"(
+          Check that the overlay filesystem is correctly mounted.
+
+          Nix does not manage the overlayfs mount point itself, but the correct
+          functioning of the overlay store does depend on this mount point being set up
+          correctly. Rather than just assume this is the case, check that the lowerdir
+          and upperdir options are what we expect them to be. This check is on by
+          default, but can be disabled if needed.
+        )"};
+
+    const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook",
+        R"(
+          Script or other executable to run when overlay filesystem needs remounting.
+
+          This is occasionally necessary when deleting a store path that exists in both upper and lower layers.
+          In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly
+          is the only way to perform the deletion without creating a "whiteout".
+          However this causes the OverlayFS kernel data structures to get out-of-sync,
+          and can lead to 'stale file handle' errors; remounting solves the problem.
+
+          The store directory is passed as an argument to the invoked executable.
+        )"};
+
+    const std::string name() override { return "Experimental Local Overlay Store"; }
+
+    std::optional<ExperimentalFeature> experimentalFeature() const override
+    {
+        return ExperimentalFeature::LocalOverlayStore;
+    }
+
+    std::string doc() override;
+
+protected:
+    /**
+     * @return The host OS path corresponding to the store path for the
+     * upper layer.
+     *
+     * @note The there is no guarantee a store object is actually stored
+     * at that file path. It might be stored in the lower layer instead,
+     * or it might not be part of this store at all.
+     */
+    Path toUpperPath(const StorePath & path);
+};
+
+/**
+ * Variation of local store using OverlayFS for the store directory.
+ *
+ * Documentation on overridden methods states how they differ from their
+ * `LocalStore` counterparts.
+ */
+class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore
+{
+    /**
+     * The store beneath us.
+     *
+     * Our store dir should be an overlay fs where the lower layer
+     * is that store's store dir, and the upper layer is some
+     * scratch storage just for us.
+     */
+    ref<LocalFSStore> lowerStore;
+
+public:
+    LocalOverlayStore(const Params & params);
+
+    LocalOverlayStore(std::string scheme, std::string path, const Params & params)
+        : LocalOverlayStore(params)
+    {
+        if (!path.empty())
+            throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params");
+    }
+
+    static std::set<std::string> uriSchemes()
+    {
+        return { "local-overlay" };
+    }
+
+    std::string getUri() override
+    {
+        return "local-overlay://";
+    }
+
+private:
+    /**
+     * First copy up any lower store realisation with the same key, so we
+     * merge rather than mask it.
+     */
+    void registerDrvOutput(const Realisation & info) override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     */
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     *
+     * In addition, copy up metadata for lower store objects (and their
+     * closure). (I.e. Optimistically cache in the upper DB.)
+     */
+    bool isValidPathUncached(const StorePath & path) override;
+
+    /**
+     * Check the lower store and upper DB.
+     */
+    void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
+
+    /**
+     * Check the lower store and upper DB.
+     */
+    StorePathSet queryValidDerivers(const StorePath & path) override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     */
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
+
+    /**
+     * First copy up any lower store realisation with the same key, so we
+     * merge rather than mask it.
+     */
+    void registerValidPaths(const ValidPathInfos & infos) override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     */
+    void queryRealisationUncached(const DrvOutput&,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
+
+    /**
+     * Call `remountIfNecessary` after collecting garbage normally.
+     */
+    void collectGarbage(const GCOptions & options, GCResults & results) override;
+
+    /**
+     * Check which layers the store object exists in to try to avoid
+     * needing to remount.
+     */
+    void deleteStorePath(const Path & path, uint64_t & bytesFreed) override;
+
+    /**
+     * Deduplicate by removing store objects from the upper layer that
+     * are now in the lower layer.
+     *
+     * Operations on a layered store will not cause duplications, but addition of
+     * new store objects to the lower layer can instill induce them
+     * (there is no way to prevent that). This cleans up those
+     * duplications.
+     *
+     * @note We do not yet optomise the upper layer in the normal way
+     * (hardlink) yet. We would like to, but it requires more
+     * refactoring of existing code to support this sustainably.
+     */
+    void optimiseStore() override;
+
+    /**
+     * Check all paths registered in the upper DB.
+     *
+     * Note that this includes store objects that reside in either overlayfs layer;
+     * just enumerating the contents of the upper layer would skip them.
+     *
+     * We don't verify the contents of both layers on the assumption that the lower layer is far bigger,
+     * and also the observation that anything not in the upper db the overlayfs doesn't yet care about.
+     */
+    VerificationResult verifyAllValidPaths(RepairFlag repair) override;
+
+    /**
+     * Deletion only effects the upper layer, so we ignore lower-layer referrers.
+     */
+    void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override;
+
+    /**
+     * Call the `remountHook` if we have done something such that the
+     * OverlayFS needed to be remounted. See that hook's user-facing
+     * documentation for further details.
+     */
+    void remountIfNecessary();
+
+    /**
+     * State for `remountIfNecessary`
+     */
+    std::atomic_bool _remountRequired = false;
+};
+
+}

src/libstore/local-overlay-store.md

@@ -0,0 +1,123 @@
+R"(
+
+**Store URL format**: `local-overlay`
+
+This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short).
+Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store].
+("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.)
+To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly).
+
+### Conceptual parts of a local overlay store
+
+*This is a more abstract/conceptual description of the parts of a layered store, an authoritative reference.
+For more "practical" instructions, see the worked-out example in the next subsection.*
+
+The parts of a local overlay store are as follows:
+
+- **Lower store**:
+
+  This is any store implementation that includes a store directory as part of the native operating system filesystem.
+  For example, this could be a [local store], [local daemon store], or even another local overlay store.
+
+  The local overlay store never tries to modify the lower store in any way.
+  Something else could modify the lower store, but there are restrictions on this
+  Nix itself requires that this store only grow, and not change in other ways.
+  For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects.
+  (In addition, the underlying filesystem overlay mechanism may impose additional restrictions, see below.)
+
+  The lower store must not change while it is mounted as part of an overlay store.
+  To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`).
+
+  Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting.
+
+  - **Lower store directory**:
+
+    This is the directory used/exposed by the lower store.
+
+    Specified with `lower-store.real` setting.
+
+    As specified above, Nix requires the local store can only grow not change in other ways.
+    Linux's OverlayFS in addition imposes the further requirement that this directory cannot change at all.
+    That means that, while any local overlay store exists that is using this store as a lower store, this directory must not change.
+
+  - **Lower metadata source**:
+
+    This is abstract, just some way to read the metadata of lower store [store objects][store object].
+    For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]).
+
+    This need not be writable.
+    As stated above a local overlay store never tries to modify its lower store.
+    The lower store's metadata is considered part of the lower store, just as the store's [file system objects][file system object] that appear in the store directory are.
+
+- **Upper almost-store**:
+
+  This is almost but not quite just a [local store].
+  That is because taken in isolation, not as part of a local overlay store, by itself, it would appear corrupted.
+  But combined with everything else as part of an overlay local store, it is valid.
+
+  - **Upper layer directory**:
+
+    This contains additional [store objects][store object]
+    (or, strictly speaking, their [file system objects][file system object] that the local overlay store will extend the lower store with).
+
+    Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting.
+
+  - **Upper store directory**:
+
+    This contains all the store objects from each of the two directories.
+
+    The lower store directory and upper layer directory are combined via OverlayFS to create this directory.
+    Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first.
+    Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings.
+
+    Specified with the [`real`](#store-experimental-local-overlay-store-real) setting.
+
+  - **Upper SQLite database**:
+
+    This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta.
+    The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer.
+    (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.)
+
+    The location of the database is directly specified, but depends on the [`state`](#store-experimental-local-overlay-store-state) setting.
+    It is is always `${state}/db`.
+
+[file system object]: @docroot@/store/file-system-object.md
+[store object]: @docroot@/store/store-object.md
+
+
+### Example filesystem layout
+
+Here is a worked out example of usage, following the concepts in the previous section.
+
+Say we have the following paths:
+
+- `/mnt/example/merged-store/nix/store`
+
+- `/mnt/example/store-a/nix/store`
+
+- `/mnt/example/store-b`
+
+Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`:
+
+```
+local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b
+```
+
+The lower store directory is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`.
+
+Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly:
+
+```shell
+mount -t overlay overlay \
+  -o lowerdir="/mnt/example/store-a/nix/store" \
+  -o upperdir="/mnt/example/store-b" \
+  -o workdir="/mnt/example/workdir" \
+  "/mnt/example/merged-store/nix/store"
+```
+
+Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`.
+
+By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not.
+You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to.
+
+)"

src/libstore/local-store.cc

@@ -465,6 +465,12 @@ AutoCloseFD LocalStore::openGCLock()
 }
 
 
+void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed)
+{
+    deletePath(path, bytesFreed);
+}
+
+
 LocalStore::~LocalStore()
 {
     std::shared_future<void> future;
@@ -1369,40 +1375,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 {
     printInfo("reading the Nix store...");
 
-    bool errors = false;
-
     /* Acquire the global GC lock to get a consistent snapshot of
        existing and valid paths. */
     auto fdGCLock = openGCLock();
     FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
 
-    StorePathSet validPaths;
-
-    {
-        StorePathSet storePathsInStoreDir;
-        /* Why aren't we using `queryAllValidPaths`? Because that would
-           tell us about all the paths than the database knows about. Here we
-           want to know about all the store paths in the store directory,
-           regardless of what the database thinks.
-
-           We will end up cross-referencing these two sources of truth (the
-           database and the filesystem) in the loop below, in order to catch
-           invalid states.
-         */
-        for (auto & i : readDirectory(realStoreDir)) {
-            try {
-                storePathsInStoreDir.insert({i.name});
-            } catch (BadStorePath &) { }
-        }
-
-        /* Check whether all valid paths actually exist. */
-        printInfo("checking path existence...");
-
-        StorePathSet done;
-
-        for (auto & i : queryAllValidPaths())
-            verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors);
-    }
+    auto [errors, validPaths] = verifyAllValidPaths(repair);
 
     /* Optionally, check the content hashes (slow). */
     if (checkContents) {
@@ -1491,21 +1469,61 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 }
 
 
-void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir,
+LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair)
+{
+    StorePathSet storePathsInStoreDir;
+    /* Why aren't we using `queryAllValidPaths`? Because that would
+       tell us about all the paths than the database knows about. Here we
+       want to know about all the store paths in the store directory,
+       regardless of what the database thinks.
+
+       We will end up cross-referencing these two sources of truth (the
+       database and the filesystem) in the loop below, in order to catch
+       invalid states.
+     */
+    for (auto & i : readDirectory(realStoreDir)) {
+        try {
+            storePathsInStoreDir.insert({i.name});
+        } catch (BadStorePath &) { }
+    }
+
+    /* Check whether all valid paths actually exist. */
+    printInfo("checking path existence...");
+
+    StorePathSet done;
+
+    auto existsInStoreDir = [&](const StorePath & storePath) {
+        return storePathsInStoreDir.count(storePath);
+    };
+
+    bool errors = false;
+    StorePathSet validPaths;
+
+    for (auto & i : queryAllValidPaths())
+        verifyPath(i, existsInStoreDir, done, validPaths, repair, errors);
+
+    return {
+        .errors = errors,
+        .validPaths = validPaths,
+    };
+}
+
+
+void LocalStore::verifyPath(const StorePath & path, std::function<bool(const StorePath &)> existsInStoreDir,
     StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors)
 {
     checkInterrupt();
 
     if (!done.insert(path).second) return;
 
-    if (!storePathsInStoreDir.count(path)) {
+    if (!existsInStoreDir(path)) {
         /* Check any referrers first.  If we can invalidate them
            first, then we can invalidate this path as well. */
         bool canInvalidate = true;
         StorePathSet referrers; queryReferrers(path, referrers);
         for (auto & i : referrers)
             if (i != path) {
-                verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors);
+                verifyPath(i, existsInStoreDir, done, validPaths, repair, errors);
                 if (validPaths.count(i))
                     canInvalidate = false;
             }

src/libstore/local-store.hh

@@ -229,6 +229,25 @@ public:
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
 
+    /**
+     * Called by `collectGarbage` to trace in reverse.
+     *
+     * Using this rather than `queryReferrers` directly allows us to
+     * fine-tune which referrers we consider for garbage collection;
+     * some store implementations take advantage of this.
+     */
+    virtual void queryGCReferrers(const StorePath & path, StorePathSet & referrers)
+    {
+        return queryReferrers(path, referrers);
+    }
+
+    /**
+     * Called by `collectGarbage` to recursively delete a path.
+     * The default implementation simply calls `deletePath`, but it can be
+     * overridden by stores that wish to provide their own deletion behaviour.
+     */
+    virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed);
+
     /**
      * Optimise the disk space usage of the Nix store by hard-linking
      * files with the same contents.
@@ -245,6 +264,31 @@ public:
 
     bool verifyStore(bool checkContents, RepairFlag repair) override;
 
+protected:
+
+    /**
+     * Result of `verifyAllValidPaths`
+     */
+    struct VerificationResult {
+        /**
+         * Whether any errors were encountered
+         */
+        bool errors;
+
+        /**
+         * A set of so-far valid paths. The store objects pointed to by
+         * those paths are suitable for further validation checking.
+         */
+        StorePathSet validPaths;
+    };
+
+    /**
+     * First, unconditional step of `verifyStore`
+     */
+    virtual VerificationResult verifyAllValidPaths(RepairFlag repair);
+
+public:
+
     /**
      * Register the validity of a path, i.e., that `path` exists, that
      * the paths referenced by it exists, and in the case of an output
@@ -255,7 +299,7 @@ public:
      */
     void registerValidPath(const ValidPathInfo & info);
 
-    void registerValidPaths(const ValidPathInfos & infos);
+    virtual void registerValidPaths(const ValidPathInfos & infos);
 
     unsigned int getProtocol() override;
 
@@ -290,6 +334,11 @@ public:
 
     std::optional<std::string> getVersion() override;
 
+protected:
+
+    void verifyPath(const StorePath & path, std::function<bool(const StorePath &)> existsInStoreDir,
+        StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
+
 private:
 
     /**
@@ -313,9 +362,6 @@ private:
      */
     void invalidatePathChecked(const StorePath & path);
 
-    void verifyPath(const StorePath & path, const StorePathSet & store,
-        StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
-
     std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(State & state, const StorePath & path);
 
     void updatePathInfo(State & state, const ValidPathInfo & info);

src/libstore/store-api.cc

@@ -1402,6 +1402,7 @@ ref<Store> openStore(const std::string & uri_,
         params.insert(uriParams.begin(), uriParams.end());
 
         if (auto store = openFromNonUri(uri, params)) {
+            experimentalFeatureSettings.require(store->experimentalFeature());
             store->warnUnknownSettings();
             return ref<Store>(store);
         }

src/libutil/experimental-features.cc

@@ -262,6 +262,14 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         )",
         .trackingUrl = "https://github.com/NixOS/nix/milestone/46",
     },
+    {
+        .tag = Xp::LocalOverlayStore,
+        .name = "local-overlay-store",
+        .description = R"(
+            Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store).
+        )",
+        .trackingUrl = "https://github.com/NixOS/nix/milestone/50",
+    },
     {
         .tag = Xp::ConfigurableImpureEnv,
         .name = "configurable-impure-env",

src/libutil/experimental-features.hh

@@ -32,6 +32,7 @@ enum struct ExperimentalFeature
     DynamicDerivations,
     ParseTomlTimestamps,
     ReadOnlyLocalStore,
+    LocalOverlayStore,
     ConfigurableImpureEnv,
     MountedSSHStore,
     VerifiedFetches,

tests/functional/hermetic.nix

@@ -1,4 +1,9 @@
-{ busybox, seed }:
+{ busybox
+, seed
+# If we want the final derivation output to have references to its
+# dependencies. Some tests need/want this, other don't.
+, withFinalRefs ? false
+}:
 
 with import ./config.nix;
 
@@ -40,7 +45,7 @@ let
     buildCommand = ''
       echo hi-input3
       read x < ${input2}
-      echo $x BAZ > $out
+      echo ${input2} $x BAZ > $out
     '';
   };
 
@@ -54,6 +59,6 @@ in
       ''
         read x < ${input1}
         read y < ${input3}
-        echo "$x $y" > $out
+        echo ${if (builtins.trace withFinalRefs withFinalRefs) then "${input1} ${input3}" else ""} "$x $y" > $out
       '';
   }

tests/functional/init.sh

@@ -3,7 +3,7 @@ source common/vars-and-functions.sh
 
 test -n "$TEST_ROOT"
 if test -d "$TEST_ROOT"; then
-    chmod -R u+w "$TEST_ROOT"
+    chmod -R u+rw "$TEST_ROOT"
     # We would delete any daemon socket, so let's stop the daemon first.
     killDaemon
     rm -rf "$TEST_ROOT"

tests/functional/local-overlay-store/add-lower-inner.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Add something to the overlay store
+overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store")
+stat "$storeBRoot/$overlayPath"
+
+# Now add something to the lower store
+lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store")
+stat "$storeVolume/store-a/$lowerPath"
+
+# Remount overlayfs to ensure synchronization
+remountOverlayfs
+
+# Path should be accessible via overlay store
+stat "$storeBRoot/$lowerPath"

tests/functional/local-overlay-store/add-lower.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./add-lower-inner.sh

tests/functional/local-overlay-store/bad-uris.sh

@@ -0,0 +1,25 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+setupStoreDirs
+
+mkdir -p $TEST_ROOT/bad_test
+badTestRoot=$TEST_ROOT/bad_test
+storeBadRoot="local-overlay://?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop"
+storeBadLower="local-overlay://?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop"
+storeBadUpper="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot"
+
+declare -a storesBad=(
+    "$storeBadRoot" "$storeBadLower" "$storeBadUpper"
+)
+
+for i in "${storesBad[@]}"; do
+    echo $i
+    unshare --mount --map-root-user bash <<EOF
+        source common.sh
+        setupStoreDirs
+        mountOverlayfs
+        expectStderr 1 nix doctor --store "$i" | grepQuiet "overlay filesystem .* mounted incorrectly"
+EOF
+done

tests/functional/local-overlay-store/build-inner.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+### Do a build in overlay store
+
+path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link)
+
+# Checking for path in lower layer (should fail)
+expect 1 stat $(toRealPath "$storeA/nix/store" "$path")
+
+# Checking for path in upper layer
+stat $(toRealPath "$storeBTop" "$path")
+
+# Verifying path in overlay store
+nix-store --verify-path --store "$storeB" "$path"

tests/functional/local-overlay-store/build.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./build-inner.sh

tests/functional/local-overlay-store/check-post-init-inner.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+### Check status
+
+# Checking for path in lower layer
+stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore")
+
+# Checking for path in upper layer (should fail)
+expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore")
+
+# Checking for path in overlay store matching lower layer
+diff $(toRealPath "$storeA/nix/store" "$pathInLowerStore") $(toRealPath "$storeBRoot/nix/store" "$pathInLowerStore")
+
+# Checking requisites query agreement
+[[ \
+  $(nix-store --store $storeA --query --requisites $drvPath) \
+  == \
+  $(nix-store --store $storeB --query --requisites $drvPath) \
+  ]]
+
+# Checking referrers query agreement
+busyboxStore=$(nix store --store $storeA add-path $busybox)
+[[ \
+  $(nix-store --store $storeA --query --referrers $busyboxStore) \
+  == \
+  $(nix-store --store $storeB --query --referrers $busyboxStore) \
+  ]]
+
+# Checking derivers query agreement
+[[ \
+  $(nix-store --store $storeA --query --deriver $pathInLowerStore) \
+  == \
+  $(nix-store --store $storeB --query --deriver $pathInLowerStore) \
+  ]]
+
+# Checking outputs query agreement
+[[ \
+  $(nix-store --store $storeA --query --outputs $drvPath) \
+  == \
+  $(nix-store --store $storeB --query --outputs $drvPath) \
+  ]]
+
+# Verifying path in lower layer
+nix-store --verify-path --store "$storeA" "$pathInLowerStore"
+
+# Verifying path in merged-store
+nix-store --verify-path --store "$storeB" "$pathInLowerStore"
+
+hashPart=$(echo $pathInLowerStore | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//')
+
+# Lower store can find from hash part
+[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $pathInLowerStore ]]
+
+# merged store can find from hash part
+[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $pathInLowerStore ]]

tests/functional/local-overlay-store/check-post-init.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./check-post-init-inner.sh

tests/functional/local-overlay-store/common.sh

@@ -0,0 +1,106 @@
+source ../common.sh
+
+# The new Linux mount interface does not seem to support remounting
+# OverlayFS mount points.
+#
+# It is not clear whether this is intentional or not:
+#
+# The  kernel source code [1] would seem to indicate merely remounting
+# while *changing* mount options is now an error because it erroneously
+# succeeded (by ignoring those new options) before. However, we are
+# *not* trying to remount with changed options, and are still hitting
+# the failure when using the new interface.
+#
+# For further details, see these `util-linux` issues:
+#
+#  - https://github.com/util-linux/util-linux/issues/2528
+#  - https://github.com/util-linux/util-linux/issues/2576
+#
+# In the meantime, setting this environment variable to "always" will
+# force the use of the old mount interface, keeping the remounting
+# working and these tests passing.
+#
+# [1]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/overlayfs/params.c?id=3006adf3be79cde4d14b1800b963b82b6e5572e0#n549
+export LIBMOUNT_FORCE_MOUNT2=always
+
+requireEnvironment () {
+  requireSandboxSupport
+  [[ $busybox =~ busybox ]] || skipTest "no busybox"
+  if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi
+  needLocalStore "The test uses --store always so we would just be bypassing the daemon"
+}
+
+addConfig () {
+    echo "$1" >> "$NIX_CONF_DIR/nix.conf"
+}
+
+setupConfig () {
+  addConfig "require-drop-supplementary-groups = false"
+  addConfig "build-users-group = "
+}
+
+enableFeatures "local-overlay-store"
+
+setupStoreDirs () {
+  # Attempt to create store dirs on tmpfs volume.
+  # This ensures lowerdir, upperdir and workdir will be on
+  # a consistent filesystem that fully supports OverlayFS.
+  storeVolume="$TEST_ROOT/stores"
+  mkdir -p "$storeVolume"
+  mount -t tmpfs tmpfs "$storeVolume" || true  # But continue anyway if that fails.
+
+  storeA="$storeVolume/store-a"
+  storeBTop="$storeVolume/store-b"
+  storeBRoot="$storeVolume/merged-store"
+  storeB="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop"
+  # Creating testing directories
+  mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir}
+}
+
+# Mounting Overlay Store
+mountOverlayfs () {
+  mount -t overlay overlay \
+    -o lowerdir="$storeA/nix/store" \
+    -o upperdir="$storeBTop" \
+    -o workdir="$storeVolume/workdir" \
+    "$storeBRoot/nix/store" \
+    || skipTest "overlayfs is not supported"
+
+  cleanupOverlay () {
+    umount "$storeBRoot/nix/store"
+    rm -r $storeVolume/workdir
+  }
+  trap cleanupOverlay EXIT
+}
+
+remountOverlayfs () {
+  mount -o remount "$storeBRoot/nix/store"
+}
+
+toRealPath () {
+  storeDir=$1; shift
+  storePath=$1; shift
+  echo $storeDir$(echo $storePath | sed "s^${NIX_STORE_DIR:-/nix/store}^^")
+}
+
+initLowerStore () {
+  # Init lower store with some stuff
+  nix-store --store "$storeA" --add ../dummy
+
+  # Build something in lower store
+  drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg withFinalRefs true --arg busybox "$busybox" --arg seed 1)
+  pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath)
+}
+
+execUnshare () {
+  exec unshare --mount --map-root-user "$SHELL" "$@"
+}
+
+addTextToStore() {
+  storeDir=$1; shift
+  filename=$1; shift
+  content=$1; shift
+  filePath="$TEST_HOME/$filename"
+  echo "$content" > "$filePath"
+  nix-store --store "$storeDir" --add "$filePath"
+}

tests/functional/local-overlay-store/delete-duplicate-inner.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Add to overlay before lower to ensure file is duplicated
+upperPath=$(nix-store --store "$storeB" --add delete-duplicate.sh)
+lowerPath=$(nix-store --store "$storeA" --add delete-duplicate.sh)
+[[ "$upperPath" = "$lowerPath" ]]
+
+# Check there really are two files with different inodes
+upperInode=$(stat -c %i "$storeBRoot/$upperPath")
+lowerInode=$(stat -c %i "$storeA/$lowerPath")
+[[ "$upperInode" != "$lowerInode" ]]
+
+# Now delete file via the overlay store
+nix-store --store "$storeB&remount-hook=$PWD/remount.sh" --delete "$upperPath"
+
+# Check there is no longer a file in upper layer
+expect 1 stat "$storeBTop/${upperPath##/nix/store/}"
+
+# Check that overlay file is now the one in lower layer
+upperInode=$(stat -c %i "$storeBRoot/$upperPath")
+lowerInode=$(stat -c %i "$storeA/$lowerPath")
+[[ "$upperInode" = "$lowerInode" ]]

tests/functional/local-overlay-store/delete-duplicate.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./delete-duplicate-inner.sh

tests/functional/local-overlay-store/delete-refs-inner.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+export NIX_REMOTE="$storeB"
+stateB="$storeBRoot/nix/var/nix"
+hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2)
+input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input1 -j0)
+input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input2 -j0)
+input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input3 -j0)
+
+# Can't delete because referenced
+expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path"
+expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path"
+expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path"
+
+# These same paths are referenced in the lower layer (by the seed 1
+# build done in `initLowerStore`).
+expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path"
+expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path"
+
+# Can delete
+nix-store --delete $hermetic
+
+# Now unreferenced in upper layer, can delete
+nix-store --delete $input3
+nix-store --delete $input2

tests/functional/local-overlay-store/delete-refs.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./delete-refs-inner.sh

tests/functional/local-overlay-store/gc-inner.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+export NIX_REMOTE="$storeB"
+stateB="$storeBRoot/nix/var/nix"
+outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2)
+
+# Set a GC root.
+mkdir -p "$stateB"
+rm -f "$stateB"/gcroots/foo
+ln -sf $outPath "$stateB"/gcroots/foo
+
+[ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ]
+
+nix-store --gc --print-roots | grep $outPath
+nix-store --gc --print-live | grep $outPath
+if nix-store --gc --print-dead | grep -E $outPath$; then false; fi
+
+nix-store --gc --print-dead
+
+expect 1 nix-store --delete $outPath
+test -e "$storeBRoot/$outPath"
+
+shopt -s nullglob
+for i in $storeBRoot/*; do
+    if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon
+    touch $i.lock
+    touch $i.chroot
+done
+
+nix-collect-garbage
+
+# Check that the root and its dependencies haven't been deleted.
+cat "$storeBRoot/$outPath"
+
+rm "$stateB"/gcroots/foo
+
+nix-collect-garbage
+
+# Check that the output has been GC'd.
+test ! -e $outPath
+
+# Check that the store is empty.
+[ "$(ls -1 "$storeBTop" | wc -l)" = "0" ]

tests/functional/local-overlay-store/gc.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./gc-inner.sh

tests/functional/local-overlay-store/local.mk

@@ -0,0 +1,14 @@
+local-overlay-store-tests := \
+  $(d)/check-post-init.sh \
+  $(d)/redundant-add.sh \
+  $(d)/build.sh \
+  $(d)/bad-uris.sh \
+  $(d)/add-lower.sh \
+  $(d)/delete-refs.sh \
+  $(d)/delete-duplicate.sh \
+  $(d)/gc.sh \
+  $(d)/verify.sh \
+  $(d)/optimise.sh \
+  $(d)/stale-file-handle.sh
+
+install-tests-groups += local-overlay-store

tests/functional/local-overlay-store/optimise-inner.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Create a file to add to store
+dupFilePath="$TEST_ROOT/dup-file"
+echo Duplicate > "$dupFilePath"
+
+# Add it to the overlay store (it will be written to the upper layer)
+dupFileStorePath=$(nix-store --store "$storeB" --add "$dupFilePath")
+
+# Now add it to the lower store so the store path is duplicated
+nix-store --store "$storeA" --add "$dupFilePath"
+
+# Ensure overlayfs and layers and synchronised
+remountOverlayfs
+
+dupFilename="${dupFileStorePath#/nix/store}"
+lowerPath="$storeA/$dupFileStorePath"
+upperPath="$storeBTop/$dupFilename"
+overlayPath="$storeBRoot/nix/store/$dupFilename"
+
+# Check store path exists in both layers and overlay
+lowerInode=$(stat -c %i "$lowerPath")
+upperInode=$(stat -c %i "$upperPath")
+overlayInode=$(stat -c %i "$overlayPath")
+[[ $upperInode == $overlayInode ]]
+[[ $upperInode != $lowerInode ]]
+
+# Run optimise to deduplicate store paths
+nix-store --store "$storeB" --optimise
+remountOverlayfs
+
+# Check path only exists in lower store
+stat "$lowerPath"
+stat "$overlayPath"
+expect 1 stat "$upperPath"

tests/functional/local-overlay-store/optimise.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./optimise-inner.sh

tests/functional/local-overlay-store/redundant-add-inner.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+### Do a redundant add
+
+# (Already done in `initLowerStore`, but repeated here for clarity.)
+pathInLowerStore=$(nix-store --store "$storeA" --add ../dummy)
+
+# upper layer should not have it
+expect 1 stat $(toRealPath "$storeBTop/nix/store" "$pathInLowerStore")
+
+pathFromB=$(nix-store --store "$storeB" --add ../dummy)
+
+[[ $pathInLowerStore == $pathFromB ]]
+
+# lower store should have it from before
+stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore")
+
+# upper layer should still not have it (no redundant copy)
+expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore")

tests/functional/local-overlay-store/redundant-add.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./redundant-add-inner.sh

tests/functional/local-overlay-store/remount.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+mount -o remount "$1"

tests/functional/local-overlay-store/stale-file-handle-inner.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+buildInStore () {
+    nix-build --store "$1" ../hermetic.nix --arg busybox "$busybox" --arg seed 1 --no-out-link
+}
+
+triggerStaleFileHandle () {
+    # Arrange it so there are duplicate paths
+    nix-store --store "$storeA" --gc  # Clear lower store
+    buildInStore "$storeB"  # Build into upper layer first
+    buildInStore "$storeA"  # Then build in lower store
+
+    # Duplicate paths mean GC will have to delete via upper layer
+    nix-store --store "$storeB" --gc
+
+    # Clear lower store again to force building in upper layer
+    nix-store --store "$storeA" --gc
+
+    # Now attempting to build in upper layer will fail
+    buildInStore "$storeB"
+}
+
+# Without remounting, we should encounter errors
+expectStderr 1 triggerStaleFileHandle | grepQuiet 'Stale file handle'
+
+# Configure remount-hook and reset OverlayFS
+storeB="$storeB&remount-hook=$PWD/remount.sh"
+remountOverlayfs
+
+# Now it should succeed
+triggerStaleFileHandle

tests/functional/local-overlay-store/stale-file-handle.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./stale-file-handle-inner.sh

tests/functional/local-overlay-store/verify-inner.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+setupStoreDirs
+
+initLowerStore
+
+mountOverlayfs
+
+
+## Initialise stores for test
+
+# Realise a derivation from the lower store to propagate paths to overlay DB
+nix-store --store "$storeB" --realise $drvPath
+
+# Also ensure dummy file exists in overlay DB
+dummyPath=$(nix-store --store "$storeB" --add ../dummy)
+
+# Add something to the lower store that will not be propagated to overlay DB
+lowerOnlyPath=$(addTextToStore "$storeA" lower-only "Only in lower store")
+
+# Verify should be successful at this point
+nix-store --store "$storeB" --verify --check-contents
+
+# Make a backup so we can repair later
+backupStore="$storeVolume/backup"
+mkdir "$backupStore"
+cp -ar "$storeBRoot/nix" "$backupStore"
+
+
+## Deliberately corrupt store paths
+
+# Delete one of the derivation inputs in the lower store
+inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv")
+inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/}
+rm -v "$inputDrvFullPath"
+
+# Truncate the contents of dummy file in lower store
+find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \;
+
+# Also truncate the file that only exists in lower store
+truncate -s 0 "$storeA/$lowerOnlyPath"
+
+# Ensure overlayfs is synchronised
+remountOverlayfs
+
+
+## Now test that verify and repair work as expected
+
+# Verify overlay store without attempting to repair it
+verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents)
+<<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!"
+<<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash"
+<<<"$verifyOutput" expectStderr 1 grepQuiet "$lowerOnlyPath"  # Expect no error for corrupted lower-only path
+
+# Attempt to repair using backup
+addConfig "substituters = $backupStore"
+repairOutput=$(nix-store --store "$storeB" --verify --check-contents --repair 2>&1)
+<<<"$repairOutput" grepQuiet "copying path '$inputDrvPath'"
+<<<"$repairOutput" grepQuiet "copying path '$dummyPath'"

tests/functional/local-overlay-store/verify.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./verify-inner.sh