#include "flake.hh" #include "primops.hh" #include "eval-inline.hh" #include "fetchGit.hh" #include "download.hh" #include "args.hh" #include #include #include #include namespace nix { /* Read a registry. */ std::shared_ptr readRegistry(const Path & path) { auto registry = std::make_shared(); if (!pathExists(path)) return std::make_shared(); auto json = nlohmann::json::parse(readFile(path)); auto version = json.value("version", 0); if (version != 1) throw Error("flake registry '%s' has unsupported version %d", path, version); auto flakes = json["flakes"]; for (auto i = flakes.begin(); i != flakes.end(); ++i) registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); return registry; } /* Write a registry to a file. */ void writeRegistry(const FlakeRegistry & registry, const Path & path) { nlohmann::json json; json["version"] = 1; for (auto elem : registry.entries) json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; createDirs(dirOf(path)); writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) { FlakeRef flakeRef(json["uri"]); if (!flakeRef.isImmutable()) throw Error("cannot use mutable flake '%s' in pure mode", flakeRef); LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["contentHash"])); auto nonFlakeRequires = json["nonFlakeRequires"]; for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); if (!flakeRef.isImmutable()) throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", ""))); entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); } auto requires = json["requires"]; for (auto i = requires.begin(); i != requires.end(); ++i) entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); return entry; } LockFile readLockFile(const Path & path) { LockFile lockFile; if (!pathExists(path)) return lockFile; auto json = nlohmann::json::parse(readFile(path)); auto version = json.value("version", 0); if (version != 1) throw Error("lock file '%s' has unsupported version %d", path, version); auto nonFlakeRequires = json["nonFlakeRequires"]; for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", ""))); if (!flakeRef.isImmutable()) throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path); lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); } auto requires = json["requires"]; for (auto i = requires.begin(); i != requires.end(); ++i) lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); return lockFile; } nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) { nlohmann::json json; json["uri"] = entry.ref.to_string(); json["contentHash"] = entry.narHash.to_string(SRI); for (auto & x : entry.nonFlakeEntries) { json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI); } for (auto & x : entry.flakeEntries) json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); return json; } void writeLockFile(const LockFile & lockFile, const Path & path) { nlohmann::json json; json["version"] = 1; json["nonFlakeRequires"] = nlohmann::json::object(); for (auto & x : lockFile.nonFlakeEntries) { json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI); } json["requires"] = nlohmann::json::object(); for (auto & x : lockFile.flakeEntries) json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); createDirs(dirOf(path)); writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file } std::shared_ptr EvalState::getGlobalFlakeRegistry() { std::call_once(_globalFlakeRegistryInit, [&]() { auto path = evalSettings.flakeRegistry; if (!hasPrefix(path, "/")) { CachedDownloadRequest request(evalSettings.flakeRegistry); request.name = "flake-registry.json"; request.gcRoot = true; path = getDownloader()->downloadCached(store, request).path; } _globalFlakeRegistry = readRegistry(path); }); return _globalFlakeRegistry; } Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; } std::shared_ptr getUserRegistry() { return readRegistry(getUserRegistryPath()); } std::shared_ptr getFlagRegistry(RegistryOverrides registryOverrides) { auto flagRegistry = std::make_shared(); for (auto const & x : registryOverrides) { flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); } return flagRegistry; } // This always returns a vector with flakeReg, userReg, globalReg. // If one of them doesn't exist, the registry is left empty but does exist. const Registries EvalState::getFlakeRegistries() { Registries registries; registries.push_back(getFlagRegistry(registryOverrides)); registries.push_back(getUserRegistry()); registries.push_back(getGlobalFlakeRegistry()); return registries; } static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, std::vector pastSearches = {}); FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector pastSearches) { std::string errorMsg = "found cycle in flake registries: "; for (FlakeRef oldRef : pastSearches) { errorMsg += oldRef.to_string(); if (oldRef == newRef) throw Error(errorMsg); errorMsg += " - "; } pastSearches.push_back(newRef); return lookupFlake(state, newRef, registries, pastSearches); } static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, std::vector pastSearches) { if (registries.empty() && !flakeRef.isDirect()) throw Error("indirect flake reference '%s' is not allowed", flakeRef); for (std::shared_ptr registry : registries) { auto i = registry->entries.find(flakeRef); if (i != registry->entries.end()) { auto newRef = i->second; return updateFlakeRef(state, newRef, registries, pastSearches); } auto j = registry->entries.find(flakeRef.baseRef()); if (j != registry->entries.end()) { auto newRef = j->second; newRef.ref = flakeRef.ref; newRef.rev = flakeRef.rev; return updateFlakeRef(state, newRef, registries, pastSearches); } } if (!flakeRef.isDirect()) throw Error("could not resolve flake reference '%s'", flakeRef); return flakeRef; } // Lookups happen here too static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { FlakeRef resolvedRef = lookupFlake(state, flakeRef, impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&resolvedRef.data)) { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", refData->owner, refData->repo, resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) : resolvedRef.ref ? *resolvedRef.ref : "master"); std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") url += "?access_token=" + accessToken; CachedDownloadRequest request(url); request.unpack = true; request.name = "source"; request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; auto result = getDownloader()->downloadCached(state.store, request); if (!result.etag) throw Error("did not receive an ETag header from '%s'", url); if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); FlakeRef ref(resolvedRef.baseRef()); ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); SourceInfo info(ref); info.storePath = result.storePath; info.narHash = state.store->queryPathInfo(info.storePath)->narHash; return info; } // This downloads the entire git history else if (auto refData = std::get_if(&resolvedRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"); FlakeRef ref(resolvedRef.baseRef()); ref.ref = gitInfo.ref; ref.rev = gitInfo.rev; SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; info.narHash = state.store->queryPathInfo(info.storePath)->narHash; return info; } else if (auto refData = std::get_if(&resolvedRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); FlakeRef ref(resolvedRef.baseRef()); ref.ref = gitInfo.ref; ref.rev = gitInfo.rev; SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; info.narHash = state.store->queryPathInfo(info.storePath)->narHash; return info; } else abort(); } // This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); // Guard against symlink attacks. Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); Path realFlakeFile = state.store->toRealPath(flakeFile); if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); Flake flake(flakeRef, sourceInfo); if (!pathExists(realFlakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); Value vInfo; state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack state.forceAttrs(vInfo); if (auto epoch = vInfo.attrs->get(state.symbols.create("epoch"))) { flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); if (flake.epoch > 2019) throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); } else throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); if (auto name = vInfo.attrs->get(state.sName)) flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); else throw Error("flake '%s' lacks attribute 'name'", flakeRef); if (auto description = vInfo.attrs->get(state.sDescription)) flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) { state.forceList(*(**requires).value, *(**requires).pos); for (unsigned int n = 0; n < (**requires).value->listSize(); ++n) flake.requires.push_back(FlakeRef(state.forceStringNoCtx( *(**requires).value->listElems()[n], *(**requires).pos))); } if (std::optional nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) { state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos); for (Attr attr : *(*(**nonFlakeRequires).value).attrs) { std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef); } } if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) { state.forceFunction(*(**provides).value, *(**provides).pos); flake.vProvides = (**provides).value; } else throw Error("flake '%s' lacks attribute 'provides'", flakeRef); return flake; } // Get the `NonFlake` corresponding to a `FlakeRef`. NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false) { auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; NonFlake nonFlake(flakeRef, sourceInfo); state.store->assertStorePath(nonFlake.sourceInfo.storePath); if (state.allowedPaths) state.allowedPaths->insert(nonFlake.sourceInfo.storePath); nonFlake.alias = alias; return nonFlake; } LockFile entryToLockFile(const LockFile::FlakeEntry & entry) { LockFile lockFile; lockFile.flakeEntries = entry.flakeEntries; lockFile.nonFlakeEntries = entry.nonFlakeEntries; return lockFile; } LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { LockFile::FlakeEntry entry( resolvedFlake.flake.sourceInfo.resolvedRef, resolvedFlake.flake.sourceInfo.narHash); for (auto & info : resolvedFlake.flakeDeps) entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second)); for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { LockFile::NonFlakeEntry nonEntry( nonFlake.sourceInfo.resolvedRef, nonFlake.sourceInfo.narHash); entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); } return entry; } bool allowedToWrite(HandleLockFile handle) { return handle == UpdateLockFile || handle == RecreateLockFile; } bool recreateLockFile(HandleLockFile handle) { return handle == RecreateLockFile || handle == UseNewLockFile; } bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) { if (handle == AllPure) return false; else if (handle == TopRefUsesRegistries) return isTopRef; else if (handle == UpdateLockFile) return true; else if (handle == UseUpdatedLockFile) return true; else if (handle == RecreateLockFile) return true; else if (handle == UseNewLockFile) return true; else assert(false); } ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, HandleLockFile handleLockFile, LockFile lockFile = {}, bool topRef = false) { Flake flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); ResolvedFlake deps(flake); for (auto & nonFlakeInfo : flake.nonFlakeRequires) { FlakeRef ref = nonFlakeInfo.second; auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); if (i != lockFile.nonFlakeEntries.end()) { NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); if (nonFlake.sourceInfo.narHash != i->second.narHash) throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.nonFlakeDeps.push_back(nonFlake); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update non-flake dependency '%s' in pure mode", nonFlakeInfo.first); deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first, allowedToUseRegistries(handleLockFile, false))); } } for (auto newFlakeRef : flake.requires) { auto i = lockFile.flakeEntries.find(newFlakeRef); if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); if (newResFlake.flake.sourceInfo.narHash != i->second.narHash) throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update flake dependency '%s' in pure mode", newFlakeRef.to_string()); deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile)); } } return deps; } /* Given a flake reference, recursively fetch it and its dependencies. FIXME: this should return a graph of flakes. */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); LockFile oldLockFile; if (!recreateLockFile (handleLockFile)) { // If recreateLockFile, start with an empty lockfile oldLockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack } LockFile lockFile(oldLockFile); ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true); lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake)); if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { if (auto refData = std::get_if(&topRef.data)) { writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. runProgram("git", true, { "-C", refData->path, "add", (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); } else warn("cannot write lockfile of remote flake '%s'", topRef); } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) warn("using updated lockfile without writing it to file"); } return resFlake; } void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) { resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); } void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) { // Construct the resulting attrset '{description, provides, // ...}'. This attrset is passed lazily as an argument to 'provides'. state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8); for (auto info : resFlake.flakeDeps) { const ResolvedFlake newResFlake = info.second; auto vFlake = state.allocAttr(v, newResFlake.flake.id); callFlake(state, newResFlake, *vFlake); } for (const NonFlake nonFlake : resFlake.nonFlakeDeps) { auto vNonFlake = state.allocAttr(v, nonFlake.alias); state.mkAttrs(*vNonFlake, 4); state.store->isValidPath(nonFlake.sourceInfo.storePath); mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); // FIXME: add rev, shortRev, revCount, ... } mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); auto & path = resFlake.flake.sourceInfo.storePath; state.store->isValidPath(path); mkString(*state.allocAttr(v, state.sOutPath), path, {path}); if (resFlake.flake.sourceInfo.resolvedRef.rev) { mkString(*state.allocAttr(v, state.symbols.create("rev")), resFlake.flake.sourceInfo.resolvedRef.rev->gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), resFlake.flake.sourceInfo.resolvedRef.rev->gitShortRev()); } if (resFlake.flake.sourceInfo.revCount) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.sourceInfo.revCount); auto vProvides = state.allocAttr(v, state.symbols.create("provides")); mkApp(*vProvides, *resFlake.flake.vProvides, v); v.attrs->push_back(Attr(state.symbols.create("self"), &v)); v.attrs->sort(); } // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, HandleLockFile handle, Value & v) { callFlake(state, resolveFlake(state, flakeRef, handle), v); } // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), evalSettings.pureEval ? AllPure : UseUpdatedLockFile, v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) { flakeRef = lookupFlake(state, flakeRef, registries); std::string uri; Strings args = {"clone"}; if (auto refData = std::get_if(&flakeRef.data)) { uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; args.push_back(uri); if (flakeRef.ref) { args.push_back("--branch"); args.push_back(*flakeRef.ref); } } else if (auto refData = std::get_if(&flakeRef.data)) { args.push_back(refData->uri); if (flakeRef.ref) { args.push_back("--branch"); args.push_back(*flakeRef.ref); } } if (destDir != "") args.push_back(destDir); runProgram("git", true, args); } }