Merge commit a1dd396cc02922372314c35c8035a38bfeea08df of branch 'nix'.

nix/libstore/build.cc

@@ -13,6 +13,7 @@
 #include <sstream>
 #include <algorithm>
 
+#include <limits.h>
 #include <time.h>
 #include <sys/time.h>
 #include <sys/wait.h>

nix/libstore/derivations.cc

@@ -48,7 +48,7 @@ static Path parsePath(std::istream & str)
 {
     string s = parseString(str);
     if (s.size() == 0 || s[0] != '/')
-        throw Error(format("bad path `%1%' in derivation") % s);
+        throw FormatError(format("bad path `%1%' in derivation") % s);
     return s;
 }
 
@@ -62,7 +62,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
-Derivation parseDerivation(const string & s)
+static Derivation parseDerivation(const string & s)
 {
     Derivation drv;
     std::istringstream str(s);
@@ -112,6 +112,16 @@ Derivation parseDerivation(const string & s)
 }
 
 
+Derivation readDerivation(const Path & drvPath)
+{
+    try {
+        return parseDerivation(readFile(drvPath));
+    } catch (FormatError & e) {
+        throw Error(format("error parsing derivation `%1%': %2%") % drvPath % e.msg());
+    }
+}
+
+
 static void printString(string & res, const string & s)
 {
     res += '"';
@@ -240,7 +250,7 @@ Hash hashDerivationModulo(StoreAPI & store, Derivation drv)
         Hash h = drvHashes[i->first];
         if (h.type == htUnknown) {
             assert(store.isValidPath(i->first));
-            Derivation drv2 = parseDerivation(readFile(i->first));
+            Derivation drv2 = readDerivation(i->first);
             h = hashDerivationModulo(store, drv2);
             drvHashes[i->first] = h;
         }

nix/libstore/derivations.hh

@@ -59,8 +59,8 @@ class StoreAPI;
 Path writeDerivation(StoreAPI & store,
     const Derivation & drv, const string & name, bool repair = false);
 
-/* Parse a derivation. */
-Derivation parseDerivation(const string & s);
+/* Read a derivation from a file. */
+Derivation readDerivation(const Path & drvPath);
 
 /* Print a derivation. */
 string unparseDerivation(const Derivation & drv);

nix/libstore/local-store.cc

@@ -20,6 +20,7 @@
 #include <errno.h>
 #include <stdio.h>
 #include <time.h>
+#include <grp.h>
 
 #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
 #include <sched.h>
@@ -237,7 +238,7 @@ LocalStore::LocalStore(bool reserveSpace)
     makeStoreWritable();
     createDirs(linksDir = settings.nixStore + "/.links");
     Path profilesDir = settings.nixStateDir + "/profiles";
-    createDirs(settings.nixStateDir + "/profiles");
+    createDirs(profilesDir);
     createDirs(settings.nixStateDir + "/temproots");
     createDirs(settings.nixDBPath);
     Path gcRootsDir = settings.nixStateDir + "/gcroots";
@@ -246,6 +247,32 @@ LocalStore::LocalStore(bool reserveSpace)
         createSymlink(profilesDir, gcRootsDir + "/profiles");
     }
 
+    /* Optionally, create directories and set permissions for a
+       multi-user install. */
+    if (getuid() == 0 && settings.buildUsersGroup != "") {
+
+        Path perUserDir = profilesDir + "/per-user";
+        createDirs(perUserDir);
+        if (chmod(perUserDir.c_str(), 01777) == -1)
+            throw SysError(format("could not set permissions on `%1%' to 1777") % perUserDir);
+
+        struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
+        if (!gr)
+            throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
+                % settings.buildUsersGroup);
+
+        struct stat st;
+        if (stat(settings.nixStore.c_str(), &st))
+            throw SysError(format("getting attributes of path `%1%'") % settings.nixStore);
+
+        if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != 01775) {
+            if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
+                throw SysError(format("changing ownership of path `%1%'") % settings.nixStore);
+            if (chmod(settings.nixStore.c_str(), 01775) == -1)
+                throw SysError(format("changing permissions on path `%1%'") % settings.nixStore);
+        }
+    }
+
     checkStoreNotSymlink();
 
     /* We can't open a SQLite database if the disk is full.  Since
@@ -661,7 +688,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che
        efficiently query whether a path is an output of some
        derivation. */
     if (isDerivation(info.path)) {
-        Derivation drv = parseDerivation(readFile(info.path));
+        Derivation drv = readDerivation(info.path);
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
@@ -1290,7 +1317,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
             if (isDerivation(i->path)) {
                 // FIXME: inefficient; we already loaded the
                 // derivation in addValidPath().
-                Derivation drv = parseDerivation(readFile(i->path));
+                Derivation drv = readDerivation(i->path);
                 checkDerivationOutputs(i->path, drv);
             }
 

nix/libstore/local-store.hh

@@ -6,6 +6,11 @@
 #include "util.hh"
 #include "pathlocks.hh"
 
+#if HAVE_TR1_UNORDERED_SET
+#include <tr1/unordered_set>
+#endif
+
+
 
 class sqlite3;
 class sqlite3_stmt;
@@ -29,14 +34,12 @@ struct Derivation;
 
 struct OptimiseStats
 {
-    unsigned long totalFiles;
-    unsigned long sameContents;
     unsigned long filesLinked;
     unsigned long long bytesFreed;
     unsigned long long blocksFreed;
     OptimiseStats()
     {
-        totalFiles = sameContents = filesLinked = 0;
+        filesLinked = 0;
         bytesFreed = blocksFreed = 0;
     }
 };
@@ -303,7 +306,15 @@ private:
 
     void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
 
-    void optimisePath_(OptimiseStats & stats, const Path & path);
+#if HAVE_TR1_UNORDERED_SET
+    typedef std::tr1::unordered_set<ino_t> InodeHash;
+#else
+    typedef std::set<ino_t> InodeHash;
+#endif
+
+    InodeHash loadInodeHash();
+    Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
+    void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
 
     // Internal versions that are not wrapped in retry_sqlite.
     bool isValidPath_(const Path & path);

nix/libstore/misc.cc

@@ -11,7 +11,7 @@ Derivation derivationFromPath(StoreAPI & store, const Path & drvPath)
 {
     assertStorePath(drvPath);
     store.ensurePath(drvPath);
-    return parseDerivation(readFile(drvPath));
+    return readDerivation(drvPath);
 }
 
 

nix/libstore/optimise-store.cc

@@ -40,18 +40,66 @@ struct MakeReadOnly
 };
 
 
-void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
+LocalStore::InodeHash LocalStore::loadInodeHash()
+{
+    printMsg(lvlDebug, "loading hash inodes in memory");
+    InodeHash inodeHash;
+
+    AutoCloseDir dir = opendir(linksDir.c_str());
+    if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
+
+    struct dirent * dirent;
+    while (errno = 0, dirent = readdir(dir)) { /* sic */
+        checkInterrupt();
+        // We don't care if we hit non-hash files, anything goes
+        inodeHash.insert(dirent->d_ino);
+    }
+    if (errno) throw SysError(format("reading directory `%1%'") % linksDir);
+
+    printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
+
+    return inodeHash;
+}
+
+
+Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
+{
+    Strings names;
+
+    AutoCloseDir dir = opendir(path.c_str());
+    if (!dir) throw SysError(format("opening directory `%1%'") % path);
+
+    struct dirent * dirent;
+    while (errno = 0, dirent = readdir(dir)) { /* sic */
+        checkInterrupt();
+
+        if (inodeHash.count(dirent->d_ino)) {
+            printMsg(lvlDebug, format("`%1%' is already linked") % dirent->d_name);
+            continue;
+        }
+
+        string name = dirent->d_name;
+        if (name == "." || name == "..") continue;
+        names.push_back(name);
+    }
+    if (errno) throw SysError(format("reading directory `%1%'") % path);
+
+    return names;
+}
+
+
+void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash)
 {
     checkInterrupt();
-    
+
     struct stat st;
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting attributes of path `%1%'") % path);
 
     if (S_ISDIR(st.st_mode)) {
-        Strings names = readDirectory(path);
+        Strings names = readDirectoryIgnoringInodes(path, inodeHash);
         foreach (Strings::iterator, i, names)
-            optimisePath_(stats, path + "/" + *i);
+            optimisePath_(stats, path + "/" + *i, inodeHash);
         return;
     }
 
@@ -71,6 +119,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
         return;
     }
 
+    /* This can still happen on top-level files */
+    if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
+        printMsg(lvlDebug, format("`%1%' is already linked, with %2% other file(s).") % path % (st.st_nlink - 2));
+        return;
+    }
+
     /* Hash the file.  Note that hashPath() returns the hash over the
        NAR serialisation, which includes the execute bit on the file.
        Thus, executable and non-executable files with the same
@@ -81,7 +135,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
     Hash hash = hashPath(htSHA256, path).first;
-    stats.totalFiles++;
     printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
 
     /* Check if this is a known hash. */
@@ -89,7 +142,10 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
 
     if (!pathExists(linkPath)) {
         /* Nope, create a hard link in the links directory. */
-        if (link(path.c_str(), linkPath.c_str()) == 0) return;
+        if (link(path.c_str(), linkPath.c_str()) == 0) {
+            inodeHash.insert(st.st_ino);
+            return;
+        }
         if (errno != EEXIST)
             throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
         /* Fall through if another process created ‘linkPath’ before
@@ -102,7 +158,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
     if (lstat(linkPath.c_str(), &stLink))
         throw SysError(format("getting attributes of path `%1%'") % linkPath);
 
-    stats.sameContents++;
     if (st.st_ino == stLink.st_ino) {
         printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
         return;
@@ -160,12 +215,13 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
 void LocalStore::optimiseStore(OptimiseStats & stats)
 {
     PathSet paths = queryAllValidPaths();
+    InodeHash inodeHash = loadInodeHash();
 
     foreach (PathSet::iterator, i, paths) {
         addTempRoot(*i);
         if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
         startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
-        optimisePath_(stats, *i);
+        optimisePath_(stats, *i, inodeHash);
     }
 }
 
@@ -173,7 +229,9 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
 void LocalStore::optimisePath(const Path & path)
 {
     OptimiseStats stats;
-    if (settings.autoOptimiseStore) optimisePath_(stats, path);
+    InodeHash inodeHash;
+
+    if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash);
 }
 
 

nix/libutil/archive.cc

@@ -104,7 +104,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
         writeString(readLink(path), sink);
     }
 
-    else throw Error(format("file `%1%' has an unknown type") % path);
+    else throw Error(format("file `%1%' has an unsupported type") % path);
 
     writeString(")", sink);
 }

nix/libutil/util.cc

@@ -1041,7 +1041,7 @@ void expect(std::istream & str, const string & s)
     char s2[s.size()];
     str.read(s2, s.size());
     if (string(s2, s.size()) != s)
-        throw Error(format("expected string `%1%'") % s);
+        throw FormatError(format("expected string `%1%'") % s);
 }
 
 

nix/libutil/util.hh

@@ -326,6 +326,8 @@ bool hasSuffix(const string & s, const string & suffix);
 /* Read string `s' from stream `str'. */
 void expect(std::istream & str, const string & s);
 
+MakeError(FormatError, Error)
+
 
 /* Read a C-style string from stream `str'. */
 string parseString(std::istream & str);