daemon: Add gzip log compression.

* nix/nix-daemon/guix-daemon.cc (GUIX_OPT_LOG_COMPRESSION): New macro. (options): Mark "disable-log-compression" as hidden and add "log-compression". (parse_opt): Handle GUIX_OPT_LOG_COMPRESSION. * nix/libstore/build.cc (DerivationGoal): Add 'gzLogFile'. (openLogFile): Initialize it when 'logCompression' is COMPRESSION_GZIP. (closeLogFile, handleChildOutput): Honor 'gzLogFile'. * nix/libstore/globals.hh (Settings)[compressLog]: Remove. [logCompression]: New field. (CompressionType): New enum. * nix/libstore/globals.cc (Settings::Settings): Initialize it. (update): Remove '_get' call for 'compressLog'. * nix/local.mk (guix_daemon_LDADD, guix_register_LDADD): Add -lz. * guix/store.scm (log-file): Handle '.gz' log files. * tests/guix-daemon.sh: Add test with '--log-compression=gzip'. * doc/guix.texi (Invoking guix-daemon): Adjust accordingly. * config-daemon.ac: Check for libz and zlib.h.
2024-11-07 07:26:13 -05:00 · 2018-01-05 17:15:41 +01:00 · 2018-01-05 17:15:41 +01:00 · 29a6866886
commit 29a6866886
parent 896fec476f
9 changed files with 127 additions and 20 deletions
--- a/config-daemon.ac
+++ b/config-daemon.ac
@ -18,6 +18,12 @@ if test "x$guix_build_daemon" = "xyes"; then
  dnl Use 64-bit file system calls so that we can support files > 2 GiB.
  AC_SYS_LARGEFILE

+  dnl Look for zlib, a required dependency.
+  AC_CHECK_LIB([z], [gzdopen], [true],
+    [AC_MSG_ERROR([Guix requires zlib.  See http://www.zlib.net/.])])
+  AC_CHECK_HEADERS([zlib.h], [true],
+    [AC_MSG_ERROR([Guix requires zlib.  See http://www.zlib.net/.])])
+
  dnl Look for libbz2, a required dependency.
  AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
    [AC_MSG_ERROR([Guix requires libbz2, which is part of bzip2.  See http://www.bzip.org/.])])
--- a/doc/guix.texi
+++ b/doc/guix.texi
@ -13,7 +13,7 @@
@set OPENPGP-SIGNING-KEY-ID 3CE464558A84FDC69DB40CFB090B11993D9AEBB5

@copying
-Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès@*
+Copyright @copyright{} 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès@*
 Copyright @copyright{} 2013, 2014, 2016 Andreas Enge@*
 Copyright @copyright{} 2013 Nikita Karetnikov@*
 Copyright @copyright{} 2014, 2015, 2016 Alex Kost@*
@ -1235,12 +1235,13 @@ processes to gain access to undeclared dependencies.  It is necessary,
 though, when @command{guix-daemon} is running under an unprivileged user
 account.

-@item --disable-log-compression
-Disable compression of the build logs.
+@item --log-compression=@var{type}
+Compress build logs according to @var{type}, one of @code{gzip},
+@code{bzip2}, or @code{none}.

 Unless @code{--lose-logs} is used, all the build logs are kept in the
@var{localstatedir}.  To save space, the daemon automatically compresses
-them with bzip2 by default.  This option disables that.
+them with bzip2 by default.

@item --disable-deduplication
@cindex deduplication
--- a/guix/store.scm
+++ b/guix/store.scm
@ -1,5 +1,5 @@
 ;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@ -1567,8 +1567,10 @@ (define (log-file store file)
                                        "/log/guix/drvs/"
                                        (string-take base 2) "/"
                                        (string-drop base 2)))
+                (log.gz  (string-append log ".gz"))
                (log.bz2 (string-append log ".bz2")))
-           (cond ((file-exists? log.bz2) log.bz2)
+           (cond ((file-exists? log.gz) log.gz)
+                 ((file-exists? log.bz2) log.bz2)
                 ((file-exists? log) log)
                 (else #f))))
        (else
--- a/nix/libstore/build.cc
+++ b/nix/libstore/build.cc
@ -31,6 +31,7 @@
 #include <pwd.h>
 #include <grp.h>

+#include <zlib.h>
 #include <bzlib.h>

 /* Includes required for chroot support. */
@ -744,6 +745,7 @@ private:

    /* File descriptor for the log file. */
    FILE * fLogFile;
+    gzFile   gzLogFile;
    BZFILE * bzLogFile;
    AutoCloseFD fdLogFile;

@ -892,6 +894,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
    , needRestart(false)
    , retrySubstitution(false)
    , fLogFile(0)
+    , gzLogFile(0)
    , bzLogFile(0)
    , useChroot(false)
    , buildMode(buildMode)
@ -2599,8 +2602,25 @@ Path DerivationGoal::openLogFile()
    Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str();
    createDirs(dir);

-    if (settings.compressLog) {
+    switch (settings.logCompression)
+      {
+      case COMPRESSION_GZIP: {
+        Path logFileName = (format("%1%/%2%.gz") % dir % string(baseName, 2)).str();
+        AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
+        if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
+        closeOnExec(fd);

+	/* Note: FD will be closed by 'gzclose'.  */
+        if (!(gzLogFile = gzdopen(fd.borrow(), "w")))
+            throw Error(format("cannot open compressed log file `%1%'") % logFileName);
+
+        gzbuffer(gzLogFile, 32768);
+        gzsetparams(gzLogFile, Z_BEST_COMPRESSION, Z_DEFAULT_STRATEGY);
+
+        return logFileName;
+      }
+
+      case COMPRESSION_BZIP2: {
        Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str();
        AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
        if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
@ -2614,8 +2634,9 @@ Path DerivationGoal::openLogFile()
            throw Error(format("cannot open compressed log file `%1%'") % logFileName);

        return logFileName;
+      }

-    } else {
+      case COMPRESSION_NONE: {
        Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str();
        fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
        if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
@ -2624,10 +2645,19 @@ Path DerivationGoal::openLogFile()
      }
    }

+    abort();
+}
+

 void DerivationGoal::closeLogFile()
 {
-    if (bzLogFile) {
+    if (gzLogFile) {
+	int err;
+	err = gzclose(gzLogFile);
+	gzLogFile = NULL;
+	if (err != Z_OK) throw Error(format("cannot close compressed log file (gzip error = %1%)") % err);
+    }
+    else if (bzLogFile) {
        int err;
        BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0);
        bzLogFile = 0;
@ -2695,7 +2725,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
        }
        if (verbosity >= settings.buildVerbosity)
            writeToStderr(data);
-        if (bzLogFile) {
+
+	if (gzLogFile) {
+	    if (data.size() > 0) {
+		int count, err;
+		count = gzwrite(gzLogFile, data.data(), data.size());
+		if (count == 0) throw Error(format("cannot write to compressed log file (gzip error = %1%)") % gzerror(gzLogFile, &err));
+	    }
+	} else if (bzLogFile) {
            int err;
            BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
            if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err);
--- a/nix/libstore/globals.cc
+++ b/nix/libstore/globals.cc
@ -45,7 +45,7 @@ Settings::Settings()
    useSshSubstituter = false;
    impersonateLinux26 = false;
    keepLog = true;
-    compressLog = true;
+    logCompression = COMPRESSION_BZIP2;
    maxLogSize = 0;
    cacheFailure = false;
    pollInterval = 5;
@ -162,7 +162,7 @@ void Settings::update()
    _get(useChroot, "build-use-chroot");
    _get(impersonateLinux26, "build-impersonate-linux-26");
    _get(keepLog, "build-keep-log");
-    _get(compressLog, "build-compress-log");
+    // _get(logCompression, "build-log-compression");
    _get(maxLogSize, "build-max-log-size");
    _get(cacheFailure, "build-cache-failure");
    _get(pollInterval, "build-poll-interval");
--- a/nix/libstore/globals.hh
+++ b/nix/libstore/globals.hh
@ -8,6 +8,12 @@

 namespace nix {

+enum CompressionType
+{
+    COMPRESSION_NONE = 0,
+    COMPRESSION_GZIP = 1,
+    COMPRESSION_BZIP2 = 2
+};

 struct Settings {

@ -169,7 +175,7 @@ struct Settings {
    bool keepLog;

    /* Whether to compress logs. */
-    bool compressLog;
+    enum CompressionType logCompression;

    /* Maximum number of bytes a builder can write to stdout/stderr
       before being killed (0 means no limit). */
--- a/nix/local.mk
+++ b/nix/local.mk
@ -1,5 +1,5 @@
 # GNU Guix --- Functional package management for GNU
-# Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org>
+# Copyright © 2012, 2013, 2014, 2015, 2016, 2018 Ludovic Courtès <ludo@gnu.org>
 # Copyright © 2016 Mathieu Lirzin <mthl@gnu.org>
 #
 # This file is part of GNU Guix.
@ -132,7 +132,7 @@ guix_daemon_CPPFLAGS =				\
  -I$(top_srcdir)/%D%/libstore

 guix_daemon_LDADD =				\
-  libstore.a libutil.a libformat.a -lbz2	\
+  libstore.a libutil.a libformat.a -lz -lbz2	\
  $(SQLITE3_LIBS) $(LIBGCRYPT_LIBS)

 guix_daemon_headers =				\
@ -149,7 +149,7 @@ guix_register_CPPFLAGS =			\

 # XXX: Should we start using shared libs?
 guix_register_LDADD =				\
-  libstore.a libutil.a libformat.a -lbz2	\
+  libstore.a libutil.a libformat.a -lz -lbz2	\
  $(SQLITE3_LIBS) $(LIBGCRYPT_LIBS)


--- a/nix/nix-daemon/guix-daemon.cc
+++ b/nix/nix-daemon/guix-daemon.cc
@ -1,5 +1,5 @@
 /* GNU Guix --- Functional package management for GNU
-   Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
+   Copyright (C) 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
   Copyright (C) 2006, 2010, 2012, 2014 Eelco Dolstra <e.dolstra@tudelft.nl>

   This file is part of GNU Guix.
@ -88,6 +88,7 @@ builds derivations on behalf of its clients.");
 #define GUIX_OPT_BUILD_ROUNDS 17
 #define GUIX_OPT_TIMEOUT 18
 #define GUIX_OPT_MAX_SILENT_TIME 19
+#define GUIX_OPT_LOG_COMPRESSION 20

 static const struct argp_option options[] =
  {
@ -120,8 +121,11 @@ static const struct argp_option options[] =
      n_("build each derivation N times in a row") },
    { "lose-logs", GUIX_OPT_LOSE_LOGS, 0, 0,
      n_("do not keep build logs") },
-    { "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0, 0,
+    { "disable-log-compression", GUIX_OPT_DISABLE_LOG_COMPRESSION, 0,
+      OPTION_HIDDEN,				  // deprecated
      n_("disable compression of the build logs") },
+    { "log-compression", GUIX_OPT_LOG_COMPRESSION, "TYPE", 0,
+      n_("use the specified compression type for build logs") },

    /* '--disable-deduplication' was known as '--disable-store-optimization'
       up to Guix 0.7 included, so keep the alias around.  */
@ -197,8 +201,21 @@ parse_opt (int key, char *arg, struct argp_state *state)
 	settings.set("build-extra-chroot-dirs", chroot_dirs);
 	break;
      }
+    case GUIX_OPT_LOG_COMPRESSION:
+      if (strcmp (arg, "none") == 0)
+	settings.logCompression = COMPRESSION_NONE;
+      else if (strcmp (arg, "gzip") == 0)
+	settings.logCompression = COMPRESSION_GZIP;
+      else if (strcmp (arg, "bzip2") == 0)
+	settings.logCompression = COMPRESSION_BZIP2;
+      else
+	{
+	  fprintf (stderr, _("error: %s: unknown compression type\n"), arg);
+	  exit (EXIT_FAILURE);
+	}
+      break;
    case GUIX_OPT_DISABLE_LOG_COMPRESSION:
-      settings.compressLog = false;
+      settings.logCompression = COMPRESSION_NONE;
      break;
    case GUIX_OPT_BUILD_USERS_GROUP:
      settings.buildUsersGroup = arg;
@ -487,6 +504,8 @@ main (int argc, char *argv[])

      /* Effect all the changes made via 'settings.set'.  */
      settings.update ();
+      printMsg(lvlDebug,
+	       format ("build log compression: %1%") % settings.logCompression);

      if (settings.useSubstitutes)
 	{
--- a/tests/guix-daemon.sh
+++ b/tests/guix-daemon.sh
@ -1,5 +1,5 @@
 # GNU Guix --- Functional package management for GNU
-# Copyright © 2012, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
+# Copyright © 2012, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
 #
 # This file is part of GNU Guix.
 #
@ -193,3 +193,39 @@ do
    GUIX_DAEMON_SOCKET="$socket" guile -c "$client_code"
    kill "$daemon_pid"
 done
+
+# Log compression.
+
+guix-daemon --listen="$socket" --disable-chroot --debug --log-compression=gzip &
+daemon_pid=$!
+
+stamp="compressed-build-log-test-$$-`date +%H%M%S`"
+client_code="
+  (use-modules (guix) (gnu packages bootstrap))
+
+  (with-store store
+    (run-with-store store
+      (mlet %store-monad ((drv (lower-object
+				(computed-file \"compressed-log-test\"
+					       #~(begin
+						   (display \"$stamp\")
+                                                   (newline)
+						   (mkdir #\$output))
+					       #:guile %bootstrap-guile))))
+	(display (derivation-file-name drv))
+	(newline)
+	(return #t))))
+"
+
+GUIX_DAEMON_SOCKET="$socket"
+export GUIX_DAEMON_SOCKET
+
+drv=`guile -c "$client_code"`
+guix build "$drv"
+
+log=`guix build "$drv" --log-file`
+test -f "$log"
+case "$log" in
+    *.gz) test "`gunzip -c < "$log"`" = "$stamp" ;;
+    *)    false ;;
+esac