From 536c85ea49c16a2ecd5a1ba169975b296cd6158c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Jan 2013 15:37:52 +0100 Subject: [PATCH] Store build logs in /nix/var/log/nix/drvs/ ...where is the first two characters of the derivation. Otherwise /nix/var/log/nix/drvs may become so large that we run into all sorts of weird filesystem limits/inefficiences. For instance, ext3/ext4 filesystems will barf with "ext4_dx_add_entry:1551: Directory index full!" once you hit a few million files. --- src/libstore/build.cc | 8 ++++-- src/nix-store/nix-store.cc | 59 +++++++++++++++++++++----------------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 75802c324..d9a7f3109 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2343,13 +2343,15 @@ Path DerivationGoal::openLogFile() { if (!settings.keepLog) return ""; + string baseName = baseNameOf(drvPath); + /* Create a log file. */ - Path dir = (format("%1%/%2%") % settings.nixLogDir % drvsLogDir).str(); + Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); createDirs(dir); if (settings.compressLog) { - Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str(); + Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str(); AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName); closeOnExec(fd); @@ -2364,7 +2366,7 @@ Path DerivationGoal::openLogFile() return logFileName; } else { - Path logFileName = (format("%1%/%2%") % dir % baseNameOf(drvPath)).str(); + Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str(); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); closeOnExec(fdLogFile); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index af4f264a6..3e1946e8d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -460,35 +460,40 @@ static void opReadLog(Strings opFlags, Strings opArgs) foreach (Strings::iterator, i, opArgs) { Path path = useDeriver(followLinksToStorePath(*i)); - Path logPath = (format("%1%/%2%/%3%") % - settings.nixLogDir % drvsLogDir % baseNameOf(path)).str(); - Path logBz2Path = logPath + ".bz2"; + for (int j = 0; j < 2; j++) { + if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path); - if (pathExists(logPath)) { - /* !!! Make this run in O(1) memory. */ - string log = readFile(logPath); - writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size()); + string baseName = baseNameOf(path); + Path logPath = + j == 0 + ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() + : (format("%1%/%2%/%3%") % settings.nixLogDir % drvsLogDir % baseName).str(); + Path logBz2Path = logPath + ".bz2"; + + if (pathExists(logPath)) { + /* !!! Make this run in O(1) memory. */ + string log = readFile(logPath); + writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size()); + } + + else if (pathExists(logBz2Path)) { + AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY); + FILE * f = 0; + if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0) + throw SysError(format("opening file `%1%'") % logBz2Path); + int err; + BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0); + if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path); + unsigned char buf[128 * 1024]; + do { + int n = BZ2_bzRead(&err, bz, buf, sizeof(buf)); + if (err != BZ_OK && err != BZ_STREAM_END) + throw Error(format("error reading bzip2 file `%1%'") % logBz2Path); + writeFull(STDOUT_FILENO, buf, n); + } while (err != BZ_STREAM_END); + BZ2_bzReadClose(&err, bz); + } } - - else if (pathExists(logBz2Path)) { - AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY); - FILE * f = 0; - if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0) - throw SysError(format("opening file `%1%'") % logBz2Path); - int err; - BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0); - if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path); - unsigned char buf[128 * 1024]; - do { - int n = BZ2_bzRead(&err, bz, buf, sizeof(buf)); - if (err != BZ_OK && err != BZ_STREAM_END) - throw Error(format("error reading bzip2 file `%1%'") % logBz2Path); - writeFull(STDOUT_FILENO, buf, n); - } while (err != BZ_STREAM_END); - BZ2_bzReadClose(&err, bz); - } - - else throw Error(format("build log of derivation `%1%' is not available") % path); } }