From 36a9a6b46fd7b947542a781dc3a689ebf4a2c53b Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Wed, 7 Jun 2023 22:20:49 -0700 Subject: [PATCH] `json-meta://` store --- src/libstore/gc.cc | 2 +- src/libstore/json-meta-store.cc | 131 ++++++++++++++++++++++++++++++++ src/libstore/json-meta-store.hh | 70 +++++++++++++++++ src/libstore/json-meta-store.md | 15 ++++ src/libstore/local-log-store.cc | 31 ++++++++ src/libstore/local-log-store.hh | 32 ++++++++ src/libstore/local-store.cc | 24 ------ src/libstore/local-store.hh | 16 +--- 8 files changed, 282 insertions(+), 39 deletions(-) create mode 100644 src/libstore/json-meta-store.cc create mode 100644 src/libstore/json-meta-store.hh create mode 100644 src/libstore/json-meta-store.md create mode 100644 src/libstore/local-log-store.cc create mode 100644 src/libstore/local-log-store.hh diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 73195794a..1f650b12e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -37,7 +37,7 @@ static std::string gcSocketPath = "/gc-socket/socket"; static std::string gcRootsDir = "gcroots"; -void LocalStore::addIndirectRoot(const Path & path) +void MixLocalStore::addIndirectRoot(const Path & path) { std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); diff --git a/src/libstore/json-meta-store.cc b/src/libstore/json-meta-store.cc new file mode 100644 index 000000000..f777aea1a --- /dev/null +++ b/src/libstore/json-meta-store.cc @@ -0,0 +1,131 @@ +#include +#include +#include + +#include "archive.hh" +#include "json-meta-store.hh" +#include "callback.hh" +#include "url.hh" +#include "path-info.hh" +#include "realisation.hh" + +namespace nix { + +std::string JsonMetaStoreConfig::doc() +{ + return + "#include json-meta-store.md" + ; +} + + +JsonMetaStore::JsonMetaStore(const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , JsonMetaStoreConfig(params) + , Store(params) + , LocalFSStore(params) +{ +} + + +JsonMetaStore::JsonMetaStore( + const std::string scheme, + std::string path, + const Params & params) + : JsonMetaStore(params) +{ + if (!path.empty()) + throw UsageError("json-meta:// store url doesn't support path part, only scheme and query params"); +} + + +std::string JsonMetaStore::getUri() +{ + return *uriSchemes().begin() + "://"; +} + + +void JsonMetaStore::queryPathInfoUncached( + const StorePath & path, Callback> callback) noexcept +{ + using nlohmann::json; + + auto callbackPtr = std::make_shared(std::move(callback)); + try + { + auto info_path = metaDir.get() + "/object/" + path.hashPart() + ".json"; + try { + std::ifstream f { info_path }; + auto json = json::parse(f); + auto info = std::make_shared( + path, + UnkeyedValidPathInfo::fromJSON(*this, json)); + return (*callbackPtr)(std::move(info)); + } catch (SysError &) { + return (*callbackPtr)({}); + } + } catch (...) { + return callbackPtr->rethrow(); + } +} + + +void JsonMetaStore::queryRealisationUncached( + const DrvOutput & drvOutput, + Callback> callback) noexcept +{ + using nlohmann::json; + + auto callbackPtr = std::make_shared(std::move(callback)); + try + { + auto realisation_path = metaDir.get() + "/realisation/" + drvOutput.to_string() + ".json"; + try { + std::ifstream f { realisation_path }; + auto json = json::parse(f); + auto realisation = std::make_shared( + Realisation::fromJSON(json, realisation_path)); + return (*callbackPtr)(std::move(realisation)); + } catch (SysError &) { + return (*callbackPtr)({}); + } + } catch (...) { + return callbackPtr->rethrow(); + } +} + + +// Unimplemented methods + + +std::optional JsonMetaStore::queryPathFromHashPart( + const std::string & hashPart) +{ unsupported("queryPathFromHashPart"); } + + +void JsonMetaStore::addToStore( + const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ unsupported("addToStore"); } + + +StorePath JsonMetaStore::addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) +{ unsupported("addTextToStore"); } + + +Roots JsonMetaStore::findRoots(bool censor) +{ unsupported("findRoots"); } + + +void JsonMetaStore::collectGarbage(const GCOptions & options, GCResults & results) +{ unsupported("collectGarbage"); } + + +static RegisterStoreImplementation regJsonMetaStore; + +} diff --git a/src/libstore/json-meta-store.hh b/src/libstore/json-meta-store.hh new file mode 100644 index 000000000..635d1c1da --- /dev/null +++ b/src/libstore/json-meta-store.hh @@ -0,0 +1,70 @@ +#include "local-log-store.hh" + +namespace nix { + +/** + * Configuration for `JsonMetaStore`. + */ +struct JsonMetaStoreConfig : virtual LocalFSStoreConfig +{ + JsonMetaStoreConfig(const StringMap & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + { + } + + const PathSetting metaDir{this, + rootDir.get() ? *rootDir.get() + "/nix/var/nix/metadata" : stateDir.get() + "/metadata", + "meta", + "directory where Nix will store metadata about store object."}; + + const std::string name() override { return "Experimental Local Cache Store"; } + + std::string doc() override; +}; + +/** + * Local store that uses JSON files instead of a SQLite database. + */ +class JsonMetaStore + : public virtual JsonMetaStoreConfig + , public virtual MixLocalStore +{ + +public: + JsonMetaStore(const Params & params); + JsonMetaStore(const std::string scheme, std::string path, const Params & params); + + std::string getUri() override; + + static std::set uriSchemes() + { return { "json-meta" }; } + +private: + // Overridden methods… + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + void queryRealisationUncached( + const DrvOutput & drvOutput, + Callback> callback) noexcept override; + + std::optional queryPathFromHashPart(const std::string & hashPart) override; + + void addToStore( + const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override; + + Roots findRoots(bool censor) override; + + void collectGarbage(const GCOptions & options, GCResults & results) override; +}; + +} diff --git a/src/libstore/json-meta-store.md b/src/libstore/json-meta-store.md new file mode 100644 index 000000000..a18688090 --- /dev/null +++ b/src/libstore/json-meta-store.md @@ -0,0 +1,15 @@ +R"( + +**Store URL format**: `json-meta://?root=*root*` + +This store type persists store objects on disk without an intervening daemon exactly like the the [Local Store] does. +However, instead of storing store object metadata in a SQLite database, it stores it in JSON files in the [`meta`](...) directory. + +This is much less performant for many tasks, and not recommend for most users. +However, it does have some benefits regarding synchronization and contention. +For example, adding new store objects will not touch the JSON files for existing store objects in any way, whereas any change to the store at all with the [Local Store] will modify the SQLite database. +Thus, for certain obscure use-cases of broadcasting a store to a wide number of consumers, it may be advantageous to use a store of this type instead of a Local Store. + +[Local Store]: ./local-store.md + +)" diff --git a/src/libstore/local-log-store.cc b/src/libstore/local-log-store.cc new file mode 100644 index 000000000..4b31ed261 --- /dev/null +++ b/src/libstore/local-log-store.cc @@ -0,0 +1,31 @@ +#include "local-log-store.hh" +#include "compression.hh" + +namespace nix { + +void MixLocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + assert(drvPath.isDerivation()); + + auto baseName = drvPath.to_string(); + + auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + + if (pathExists(logPath)) return; + + createDirs(dirOf(logPath)); + + auto tmpFile = fmt("%s.tmp.%d", logPath, getpid()); + + writeFile(tmpFile, compress("bzip2", log)); + + std::filesystem::rename(tmpFile, logPath); +} + + +std::optional MixLocalStore::isTrustedClient() +{ + return Trusted; +} + +} diff --git a/src/libstore/local-log-store.hh b/src/libstore/local-log-store.hh new file mode 100644 index 000000000..b977a3988 --- /dev/null +++ b/src/libstore/local-log-store.hh @@ -0,0 +1,32 @@ +#pragma once +///@file + +#include "indirect-root-store.hh" + +namespace nix { + +/** + * Store that directly manipulates the local log directory. Probably + * will evolve to be just anything a "true" local store (SQLite or JSON) + * has in common. + * + * @todo rename `LocalStore` to `SQLiteStore`, and then rename this to + * `MixLocalStore`. `LocalFSStore` could also be renamed to + * `MixFileSystemStore`. + */ +struct MixLocalStore : virtual IndirectRootStore { + + /** + * Implementation of IndirectRootStore::addIndirectRoot(). + * + * The weak reference merely is a symlink to `path' from + * /nix/var/nix/gcroots/auto/. + */ + void addIndirectRoot(const Path & path) override; + + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + + std::optional isTrustedClient() override; +}; + +} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eafdac0cd..5bb93e434 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1552,11 +1552,6 @@ unsigned int LocalStore::getProtocol() return PROTOCOL_VERSION; } -std::optional LocalStore::isTrustedClient() -{ - return Trusted; -} - void LocalStore::vacuumDB() { @@ -1684,25 +1679,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) -{ - assert(drvPath.isDerivation()); - - auto baseName = drvPath.to_string(); - - auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); - - if (pathExists(logPath)) return; - - createDirs(dirOf(logPath)); - - auto tmpFile = fmt("%s.tmp.%d", logPath, getpid()); - - writeFile(tmpFile, compress("bzip2", log)); - - std::filesystem::rename(tmpFile, logPath); -} - std::optional LocalStore::getVersion() { return nixVersion; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 21848cc4d..cea1bafa0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -6,6 +6,7 @@ #include "pathlocks.hh" #include "store-api.hh" #include "indirect-root-store.hh" +#include "local-log-store.hh" #include "sync.hh" #include @@ -76,6 +77,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig class LocalStore : public virtual LocalStoreConfig , public virtual IndirectRootStore , public virtual GcStore + , public virtual MixLocalStore { private: @@ -214,16 +216,6 @@ private: */ Sync _fdRootsSocket; -public: - - /** - * Implementation of IndirectRootStore::addIndirectRoot(). - * - * The weak reference merely is a symlink to `path' from - * /nix/var/nix/gcroots/auto/. - */ - void addIndirectRoot(const Path & path) override; - private: void findTempRoots(Roots & roots, bool censor); @@ -310,8 +302,6 @@ public: unsigned int getProtocol() override; - std::optional isTrustedClient() override; - void vacuumDB(); void addSignatures(const StorePath & storePath, const StringSet & sigs) override; @@ -401,8 +391,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void addBuildLog(const StorePath & drvPath, std::string_view log) override; - friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal;