#include "flake.hh" #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" #include "store-api.hh" #include "fetchers/fetchers.hh" #include "finally.hh" namespace nix { using namespace flake; namespace flake { /* If 'allowLookup' is true, then resolve 'flakeRef' using the registries. */ static FlakeRef maybeLookupFlake( ref store, const FlakeRef & flakeRef, bool allowLookup) { if (!flakeRef.input->isDirect()) { if (allowLookup) return flakeRef.resolve(store); else throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef); } else return flakeRef; } typedef std::vector> FlakeCache; static FlakeRef lookupInFlakeCache( const FlakeCache & flakeCache, const FlakeRef & flakeRef) { // FIXME: inefficient. for (auto & i : flakeCache) { if (flakeRef == i.first) { debug("mapping '%s' to previously seen input '%s' -> '%s", flakeRef, i.first, i.second); return i.second; } } return flakeRef; } static std::pair fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, std::optional treeInfo, bool allowLookup, FlakeCache & flakeCache) { /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) { try { auto storePath = treeInfo->computeStorePath(*state.store); state.store->ensurePath(storePath); debug("using substituted/cached input '%s' in '%s'", originalRef, state.store->printStorePath(storePath)); auto actualPath = state.store->toRealPath(storePath); if (state.allowedPaths) state.allowedPaths->insert(actualPath); return { Tree { .actualPath = actualPath, .storePath = std::move(storePath), .info = *treeInfo, }, originalRef }; } catch (Error & e) { debug("substitution of input '%s' failed: %s", originalRef, e.what()); } } auto resolvedRef = lookupInFlakeCache(flakeCache, maybeLookupFlake(state.store, lookupInFlakeCache(flakeCache, originalRef), allowLookup)); auto [tree, lockedRef] = resolvedRef.fetchTree(state.store); debug("got tree '%s' from '%s'", state.store->printStorePath(tree.storePath), lockedRef); flakeCache.push_back({originalRef, lockedRef}); flakeCache.push_back({resolvedRef, lockedRef}); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); if (treeInfo) assert(tree.storePath == treeInfo->computeStorePath(*state.store)); return {std::move(tree), lockedRef}; } static void expectType(EvalState & state, ValueType type, Value & value, const Pos & pos) { if (value.type == tThunk && value.isTrivial()) state.forceValue(value, pos); if (value.type != type) throw Error("expected %s but got %s at %s", showType(type), showType(value.type), pos); } static std::map parseFlakeInputs( EvalState & state, Value * value, const Pos & pos); static FlakeInput parseFlakeInput(EvalState & state, const std::string & inputName, Value * value, const Pos & pos) { expectType(state, tAttrs, *value, pos); FlakeInput input { .ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}) }; auto sInputs = state.symbols.create("inputs"); auto sUrl = state.symbols.create("url"); auto sUri = state.symbols.create("uri"); // FIXME: remove soon auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); fetchers::Attrs attrs; std::optional url; for (nix::Attr attr : *(value->attrs)) { try { if (attr.name == sUrl || attr.name == sUri) { expectType(state, tString, *attr.value, *attr.pos); url = attr.value->string.s; attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, tBool, *attr.value, *attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); } else if (attr.name == sFollows) { expectType(state, tString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { state.forceValue(*attr.value); if (attr.value->type == tString) attrs.emplace(attr.name, attr.value->string.s); else throw TypeError("flake input attribute '%s' is %s while a string is expected", attr.name, showType(*attr.value)); } } catch (Error & e) { e.addPrefix(fmt("in flake attribute '%s' at '%s':\n", attr.name, *attr.pos)); throw; } } if (attrs.count("type")) try { input.ref = FlakeRef::fromAttrs(attrs); } catch (Error & e) { e.addPrefix(fmt("in flake input at '%s':\n", pos)); throw; } else { attrs.erase("url"); if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); if (url) input.ref = parseFlakeRef(*url); } return input; } static std::map parseFlakeInputs( EvalState & state, Value * value, const Pos & pos) { std::map inputs; expectType(state, tAttrs, *value, pos); for (nix::Attr & inputAttr : *(*value).attrs) { inputs.emplace(inputAttr.name, parseFlakeInput(state, inputAttr.name, inputAttr.value, *inputAttr.pos)); } return inputs; } static Flake getFlake( EvalState & state, const FlakeRef & originalRef, std::optional treeInfo, bool allowLookup, FlakeCache & flakeCache) { auto [sourceInfo, lockedRef] = fetchOrSubstituteTree( state, originalRef, treeInfo, allowLookup, flakeCache); // Guard against symlink attacks. auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); if (!isInDir(flakeFile, sourceInfo.actualPath)) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", lockedRef, state.store->printStorePath(sourceInfo.storePath)); Flake flake { .originalRef = originalRef, .lockedRef = lockedRef, .sourceInfo = std::make_shared(std::move(sourceInfo)) }; if (!pathExists(flakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack expectType(state, tAttrs, vInfo, Pos(state.symbols.create(flakeFile), 0, 0)); auto sEdition = state.symbols.create("edition"); auto sEpoch = state.symbols.create("epoch"); // FIXME: remove soon auto edition = vInfo.attrs->get(sEdition); if (!edition) edition = vInfo.attrs->get(sEpoch); if (edition) { expectType(state, tInt, *edition->value, *edition->pos); flake.edition = edition->value->integer; if (flake.edition > 201909) throw Error("flake '%s' requires unsupported edition %d; please upgrade Nix", lockedRef, flake.edition); if (flake.edition < 201909) throw Error("flake '%s' has illegal edition %d", lockedRef, flake.edition); } else throw Error("flake '%s' lacks attribute 'edition'", lockedRef); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, tString, *description->value, *description->pos); flake.description = description->value->string.s; } auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs->get(sInputs)) flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos); auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, tLambda, *outputs->value, *outputs->pos); flake.vOutputs = outputs->value; if (flake.vOutputs->lambda.fun->matchAttrs) { for (auto & formal : flake.vOutputs->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { .ref = parseFlakeRef(formal.name) }); } } } else throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); for (auto & attr : *vInfo.attrs) { if (attr.name != sEdition && attr.name != sEpoch && attr.name != state.sDescription && attr.name != sInputs && attr.name != sOutputs) throw Error("flake '%s' has an unsupported attribute '%s', at %s", lockedRef, attr.name, *attr.pos); } return flake; } Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) { FlakeCache flakeCache; return getFlake(state, originalRef, {}, allowLookup, flakeCache); } /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, it the flake is writable. */ LockedFlake lockFlake( EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) { settings.requireExperimentalFeature("flakes"); FlakeCache flakeCache; auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache); // FIXME: symlink attack auto oldLockFile = LockFile::read( flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); debug("old lock file: %s", oldLockFile); // FIXME: check whether all overrides are used. std::map overrides; for (auto & i : lockFlags.inputOverrides) overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); LockFile newLockFile; std::vector parents; std::map follows; std::function node, const InputPath & inputPathPrefix, std::shared_ptr oldNode)> computeLocks; computeLocks = [&]( const FlakeInputs & flakeInputs, std::shared_ptr node, const InputPath & inputPathPrefix, std::shared_ptr oldNode) { debug("computing lock file node '%s'", concatStringsSep("/", inputPathPrefix)); /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ // FIXME: check this for (auto & [id, input] : flake.inputs) { for (auto & [idOverride, inputOverride] : input.overrides) { auto inputPath(inputPathPrefix); inputPath.push_back(id); inputPath.push_back(idOverride); overrides.insert_or_assign(inputPath, inputOverride); } } /* Go over the flake inputs, resolve/fetch them if necessary (i.e. if they're new or the flakeref changed from what's in the lock file). */ for (auto & [id, input2] : flakeInputs) { auto inputPath(inputPathPrefix); inputPath.push_back(id); auto inputPathS = concatStringsSep("/", inputPath); debug("computing input '%s'", concatStringsSep("/", inputPath)); /* Do we have an override for this input from one of the ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); auto & input = hasOverride ? i->second : input2; /* Resolve 'follows' later (since it may refer to an input path we haven't processed yet. */ if (input.follows) { if (hasOverride) /* 'follows' from an override is relative to the root of the graph. */ follows.insert_or_assign(inputPath, *input.follows); else { /* Otherwise, it's relative to the current flake. */ InputPath path(inputPathPrefix); for (auto & i : *input.follows) path.push_back(i); follows.insert_or_assign(inputPath, path); } continue; } /* Do we have an entry in the existing lock file? And we don't have a --update-input flag for this input? */ std::shared_ptr oldLock; if (oldNode && !lockFlags.inputUpdates.count(inputPath)) { auto oldLockIt = oldNode->inputs.find(id); if (oldLockIt != oldNode->inputs.end()) oldLock = std::dynamic_pointer_cast(oldLockIt->second); } if (oldLock && oldLock->originalRef == input.ref && !hasOverride) { debug("keeping existing input '%s'", inputPathS); /* Copy the input from the old lock since its flakeref didn't change and there is no override from a higher level flake. */ auto childNode = std::make_shared( oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake); node->inputs.insert_or_assign(id, childNode); /* If we have an --update-input flag for an input of this input, then we must fetch the flake to to update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto hasChildUpdate = lb != lockFlags.inputUpdates.end() && lb->size() > inputPath.size() && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); if (hasChildUpdate) { auto inputFlake = getFlake( state, oldLock->lockedRef, oldLock->info, false, flakeCache); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); } else { /* No need to fetch this flake, we can be lazy. However there may be new overrides on the inputs of this flake, so we need to check those. */ FlakeInputs fakeInputs; for (auto & i : oldLock->inputs) { auto lockedNode = std::dynamic_pointer_cast(i.second); // Note: this node is not locked in case // of a circular reference back to the root. if (lockedNode) fakeInputs.emplace(i.first, FlakeInput { .ref = lockedNode->originalRef }); else { InputPath path(inputPath); path.push_back(i.first); follows.insert_or_assign(path, InputPath()); } } computeLocks(fakeInputs, childNode, inputPath, oldLock); } } else { /* We need to create a new lock file entry. So fetch this input. */ if (!lockFlags.allowMutable && !input.ref.input->isImmutable()) throw Error("cannot update flake input '%s' in pure mode", inputPathS); if (input.isFlake) { auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache); auto childNode = std::make_shared( inputFlake.lockedRef, inputFlake.originalRef, inputFlake.sourceInfo->info); node->inputs.insert_or_assign(id, childNode); /* Guard against circular flake imports. */ for (auto & parent : parents) if (parent == input.ref) throw Error("found circular import of flake '%s'", parent); parents.push_back(input.ref); Finally cleanup([&]() { parents.pop_back(); }); /* Recursively process the inputs of this flake. Also, unless we already have this flake in the top-level lock file, use this flake's own lock file. */ computeLocks( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); } else { auto [sourceInfo, lockedRef] = fetchOrSubstituteTree( state, input.ref, {}, lockFlags.useRegistries, flakeCache); node->inputs.insert_or_assign(id, std::make_shared(lockedRef, input.ref, sourceInfo.info, false)); } } } }; computeLocks( flake.inputs, newLockFile.root, {}, lockFlags.recreateLockFile ? nullptr : oldLockFile.root); /* Insert edges for 'follows' overrides. */ for (auto & [from, to] : follows) { debug("adding 'follows' node from '%s' to '%s'", concatStringsSep("/", from), concatStringsSep("/", to)); assert(!from.empty()); InputPath fromParent(from); fromParent.pop_back(); auto fromParentNode = newLockFile.root->findInput(fromParent); assert(fromParentNode); auto toNode = newLockFile.root->findInput(to); if (!toNode) throw Error("flake input '%s' follows non-existent flake input '%s'", concatStringsSep("/", from), concatStringsSep("/", to)); fromParentNode->inputs.insert_or_assign(from.back(), toNode); } debug("new lock file: %s", newLockFile); /* Check whether we need to / can write the new lock file. */ if (!(newLockFile == oldLockFile)) { auto diff = diffLockFiles(oldLockFile, newLockFile); if (!(oldLockFile == LockFile())) printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff)); if (lockFlags.writeLockFile) { if (auto sourcePath = topRef.input->getSourcePath()) { if (!newLockFile.isImmutable()) { if (settings.warnDirty) warn("will not write lock file of flake '%s' because it has a mutable input", topRef); } else { if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto path = *sourcePath + "/" + relPath; bool lockFileExists = pathExists(path); if (lockFileExists) warn("updating lock file '%s'", path); else warn("creating lock file '%s'", path); newLockFile.write(path); topRef.input->markChangedFile( (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", lockFlags.commitLockFile ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", relPath, lockFileExists ? "Update" : "Add", diff)) : std::nullopt); /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; FlakeCache dummyCache; flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache); if (lockFlags.commitLockFile && flake.lockedRef.input->getRev() && prevLockedRef.input->getRev() != flake.lockedRef.input->getRev()) warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev()); /* Make sure that we picked up the change, i.e. the tree should usually be dirty now. Corner case: we could have reverted from a dirty to a clean tree! */ if (flake.lockedRef.input == prevLockedRef.input && !flake.lockedRef.input->isImmutable()) throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); } } else throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); } else warn("not writing modified lock file of flake '%s'", topRef); } return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; } void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) { auto vLocks = state.allocValue(); auto vRootSrc = state.allocValue(); auto vRootSubdir = state.allocValue(); auto vTmp1 = state.allocValue(); auto vTmp2 = state.allocValue(); mkString(*vLocks, lockedFlake.lockFile.to_string()); emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); static Value * vCallFlake = nullptr; if (!vCallFlake) { vCallFlake = state.allocValue(); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" , "/"), *vCallFlake); } state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { callFlake(state, lockFlake(state, parseFlakeRef(state.forceStringNoCtx(*args[0], pos)), LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, .allowMutable = !evalSettings.pureEval, }), v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); } Fingerprint LockedFlake::getFingerprint() const { // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%d;%d;%s", flake.sourceInfo->storePath.to_string(), flake.sourceInfo->info.revCount.value_or(0), flake.sourceInfo->info.lastModified.value_or(0), lockFile)); } Flake::~Flake() { } }