From 02190b0fefe7b5739c0eac403264c07d15ae3635 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 14:45:03 +0100 Subject: [PATCH] Support hydra-build-products on binary cache stores --- src/hydra-queue-runner/build-result.cc | 77 ++++++++++--------------- src/hydra-queue-runner/build-result.hh | 4 +- src/hydra-queue-runner/builder.cc | 2 +- src/hydra-queue-runner/queue-monitor.cc | 2 +- 4 files changed, 34 insertions(+), 51 deletions(-) diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index f09d13a1..6b2741ea 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -2,26 +2,13 @@ #include "store-api.hh" #include "util.hh" #include "regex.hh" +#include "fs-accessor.hh" using namespace nix; -static std::tuple secureRead(Path fileName) -{ - auto fail = std::make_tuple(false, ""); - - if (!pathExists(fileName)) return fail; - - try { - /* For security, resolve symlinks. */ - fileName = canonPath(fileName, true); - if (!isInStore(fileName)) return fail; - return std::make_tuple(true, readFile(fileName)); - } catch (Error & e) { return fail; } -} - - -BuildOutput getBuildOutput(nix::ref store, const Derivation & drv) +BuildOutput getBuildOutput(nix::ref store, + nix::ref accessor, const Derivation & drv) { BuildOutput res; @@ -41,7 +28,6 @@ BuildOutput getBuildOutput(nix::ref store, const Derivation & drv) /* Get build products. */ bool explicitProducts = false; -#if 0 Regex regex( "(([a-zA-Z0-9_-]+)" // type (e.g. "doc") "[[:space:]]+" @@ -53,14 +39,16 @@ BuildOutput getBuildOutput(nix::ref store, const Derivation & drv) for (auto & output : outputs) { Path failedFile = output + "/nix-support/failed"; - if (pathExists(failedFile)) res.failed = true; + if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular) + res.failed = true; - auto file = secureRead(output + "/nix-support/hydra-build-products"); - if (!std::get<0>(file)) continue; + Path productsFile = output + "/nix-support/hydra-build-products"; + if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular) + continue; explicitProducts = true; - for (auto & line : tokenizeString(std::get<1>(file), "\n")) { + for (auto & line : tokenizeString(accessor->readFile(productsFile), "\n")) { BuildProduct product; Regex::Subs subs; @@ -73,32 +61,28 @@ BuildOutput getBuildOutput(nix::ref store, const Derivation & drv) /* Ensure that the path exists and points into the Nix store. */ + // FIXME: should we disallow products referring to other + // store paths, or that are outside the input closure? if (product.path == "" || product.path[0] != '/') continue; - try { - product.path = canonPath(product.path, true); - } catch (Error & e) { continue; } - if (!isInStore(product.path) || !pathExists(product.path)) continue; + product.path = canonPath(product.path); + if (!isInStore(product.path)) continue; - /* FIXME: check that the path is in the input closure - of the build? */ + auto st = accessor->stat(product.path); + if (st.type == FSAccessor::Type::tMissing) continue; product.name = product.path == output ? "" : baseNameOf(product.path); - struct stat st; - if (stat(product.path.c_str(), &st)) - throw SysError(format("getting status of ‘%1%’") % product.path); - - if (S_ISREG(st.st_mode)) { + if (st.type == FSAccessor::Type::tRegular) { product.isRegular = true; - product.fileSize = st.st_size; - product.sha1hash = hashFile(htSHA1, product.path); - product.sha256hash = hashFile(htSHA256, product.path); + product.fileSize = st.fileSize; + auto contents = accessor->readFile(product.path); + product.sha1hash = hashString(htSHA1, contents); + product.sha256hash = hashString(htSHA256, contents); } res.products.push_back(product); } } -#endif /* If no build products were explicitly declared, then add all outputs as a product of type "nix-build". */ @@ -110,31 +94,29 @@ BuildOutput getBuildOutput(nix::ref store, const Derivation & drv) product.subtype = output.first == "out" ? "" : output.first; product.name = storePathToName(product.path); -#if 0 - struct stat st; - if (stat(product.path.c_str(), &st)) - throw SysError(format("getting status of ‘%1%’") % product.path); - if (S_ISDIR(st.st_mode)) -#endif + auto st = accessor->stat(product.path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("getting status of ‘%1%’") % product.path); + if (st.type == FSAccessor::Type::tDirectory) res.products.push_back(product); } } -#if 0 /* Get the release name from $output/nix-support/hydra-release-name. */ for (auto & output : outputs) { Path p = output + "/nix-support/hydra-release-name"; - if (!pathExists(p)) continue; + if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue; try { - res.releaseName = trim(readFile(p)); + res.releaseName = trim(accessor->readFile(p)); } catch (Error & e) { continue; } // FIXME: validate release name } /* Get metrics. */ for (auto & output : outputs) { - auto file = secureRead(output + "/nix-support/hydra-metrics"); - for (auto & line : tokenizeString(std::get<1>(file), "\n")) { + Path metricsFile = output + "/nix-support/hydra-metrics"; + if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue; + for (auto & line : tokenizeString(accessor->readFile(metricsFile), "\n")) { auto fields = tokenizeString>(line); if (fields.size() < 2) continue; BuildMetric metric; @@ -144,7 +126,6 @@ BuildOutput getBuildOutput(nix::ref store, const Derivation & drv) res.metrics[metric.name] = metric; } } -#endif return res; } diff --git a/src/hydra-queue-runner/build-result.hh b/src/hydra-queue-runner/build-result.hh index 7fdd659b..72e8b4df 100644 --- a/src/hydra-queue-runner/build-result.hh +++ b/src/hydra-queue-runner/build-result.hh @@ -4,6 +4,7 @@ #include "hash.hh" #include "derivations.hh" +#include "store-api.hh" struct BuildProduct { @@ -37,4 +38,5 @@ struct BuildOutput std::map metrics; }; -BuildOutput getBuildOutput(nix::ref store, const nix::Derivation & drv); +BuildOutput getBuildOutput(nix::ref store, + nix::ref accessor, const nix::Derivation & drv); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index 487a96b1..79566c4c 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -126,7 +126,7 @@ bool State::doBuildStep(nix::ref destStore, Step::ptr step, result.errorMsg = e.msg(); } - if (result.success()) res = getBuildOutput(destStore, step->drv); + if (result.success()) res = getBuildOutput(destStore, destStore->getFSAccessor(), step->drv); } time_t stepStopTime = time(0); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index f7e40827..b44e0836 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -159,7 +159,7 @@ bool State::getQueuedBuilds(Connection & conn, ref localStore, all valid. So we mark this as a finished, cached build. */ if (!step) { Derivation drv = readDerivation(build->drvPath); - BuildOutput res = getBuildOutput(destStore, drv); + BuildOutput res = getBuildOutput(destStore, destStore->getFSAccessor(), drv); pqxx::work txn(conn); time_t now = time(0);