Merged trunk R9751 back in.

mergeTrunkBackIn.sh

@@ -1,4 +1,4 @@
-svn merge -r 9561:9584 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk
+svn merge -r 9584:9751 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk
 
 #already done:
 # 8628 
@@ -27,4 +27,4 @@ svn merge -r 9561:9584 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk
 # 9549
 # 9561
 # 9584
-
+# 9751

scripts/Makefile.am

@@ -4,7 +4,8 @@ bin_SCRIPTS = nix-collect-garbage \
   nix-pack-closure nix-unpack-closure \
   nix-copy-closure
 
-noinst_SCRIPTS = nix-profile.sh generate-patches.pl find-runtime-roots.pl
+noinst_SCRIPTS = nix-profile.sh generate-patches.pl \
+  find-runtime-roots.pl build-remote.pl
 
 nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl
 
@@ -17,6 +18,7 @@ install-exec-local: readmanifest.pm download-using-manifests.pl find-runtime-roo
 	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 
 include ../substitute.mk
@@ -32,4 +34,5 @@ EXTRA_DIST = nix-collect-garbage.in \
   generate-patches.pl.in \
   nix-pack-closure.in nix-unpack-closure.in \
   nix-copy-closure.in \
-  find-runtime-roots.pl.in
+  find-runtime-roots.pl.in \
+  build-remote.pl.in

scripts/build-remote.pl.in

@@ -0,0 +1,208 @@
+#! @perl@ -w
+
+use strict;
+use Fcntl ':flock';
+use English '-no_match_vars';
+
+# General operation:
+#
+# Try to find a free machine of type $neededSystem.  We do this as
+# follows:
+# - We acquire an exclusive lock on $currentLoad/main-lock.
+# - For each machine $machine of type $neededSystem and for each $slot
+#   less than the maximum load for that machine, we try to get an
+#   exclusive lock on $currentLoad/$machine-$slot (without blocking).
+#   If we get such a lock, we send "accept" to the caller.  Otherwise,
+#   we send "postpone" and exit. 
+# - We release the exclusive lock on $currentLoad/main-lock.
+# - We perform the build on $neededSystem.
+# - We release the exclusive lock on $currentLoad/$machine-$slot.
+#
+# The nice thing about this scheme is that if we die prematurely, the
+# locks are released automatically.
+
+my $loadIncreased = 0;
+
+my $amWilling = shift @ARGV;
+my $localSystem = shift @ARGV;
+my $neededSystem = shift @ARGV;
+my $drvPath = shift @ARGV;
+
+sub sendReply {
+    my $reply = shift;
+    open OUT, ">&3" or die;
+    print OUT "$reply\n";
+    close OUT;
+}
+
+sub decline {
+    sendReply "decline";
+    exit 0;
+}
+
+my $currentLoad = $ENV{"NIX_CURRENT_LOAD"};
+decline unless defined $currentLoad;
+mkdir $currentLoad, 0777 or die unless -d $currentLoad;
+
+my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
+decline if !defined $conf || ! -e $conf;
+
+# Decline if the local system can do the build.
+decline if $amWilling && ($localSystem eq $neededSystem);
+
+
+# Otherwise find a willing remote machine.
+my %machines;
+my %systemTypes;
+my %sshKeys;
+my %maxJobs;
+my %curJobs;
+
+
+# Read the list of machines.
+open CONF, "< $conf" or die;
+
+while (<CONF>) {
+    chomp;
+    s/\#.*$//g;
+    next if /^\s*$/;
+    /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die;
+    $machines{$1} = "";
+    $systemTypes{$1} = $2;
+    $sshKeys{$1} = $3;
+    $maxJobs{$1} = $4;
+}
+
+close CONF;
+
+
+# Acquire the exclusive lock on $currentLoad/main-lock.
+my $mainLock = "$currentLoad/main-lock";
+open MAINLOCK, ">>$mainLock" or die;
+flock(MAINLOCK, LOCK_EX) or die;
+
+
+# Find a suitable system.
+my $rightType = 0;
+my $machine;
+LOOP: foreach my $cur (keys %machines) {
+    if ($neededSystem eq $systemTypes{$cur}) {
+        $rightType = 1;
+
+        # We have a machine of the right type.  Try to get a lock on
+        # one of the machine's lock files.
+        my $slot = 0;
+        while ($slot < $maxJobs{$cur}) {
+            my $slotLock = "$currentLoad/$cur-$slot";
+            open SLOTLOCK, ">>$slotLock" or die;
+            if (flock(SLOTLOCK, LOCK_EX | LOCK_NB)) {
+                $machine = $cur;
+                last LOOP;
+            }                
+            close SLOTLOCK;
+            $slot++;
+        }
+    }
+}
+
+close MAINLOCK;
+
+
+# Didn't find one?
+if (!defined $machine) {
+    if ($rightType) {
+        sendReply "postpone";
+        exit 0;
+    } else {
+        decline;
+    }
+}
+
+# Yes we did, accept.
+sendReply "accept";
+open IN, "<&4" or die;
+my $x = <IN>;
+chomp $x;
+#print "got $x\n";  
+close IN;
+
+if ($x ne "okay") {
+    exit 0;
+}
+
+
+# Do the actual job.
+print "BUILDING REMOTE: $drvPath on $machine\n";
+
+# Make sure that we don't get any SSH passphrase or host key popups -
+# if there is any problem it should fail, not do something
+# interactive.
+$ENV{"DISPLAY"} = "";
+$ENV{"SSH_PASSWORD_FILE="} = "";
+$ENV{"SSH_ASKPASS="} = "";
+
+my $sshOpts = "-i $sshKeys{$machine} -x";
+
+# Hack to support Cygwin: if we login without a password, we don't
+# have exactly the same right as when we do.  This causes the
+# Microsoft C compiler to fail with certain flags:
+#
+#   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676
+#
+# So as a workaround, we pass a verbatim password.  ssh tries to makes
+# this very hard; the trick is to make it call SSH_ASKPASS to get the
+# password.  (It only calls this command when there is no controlling
+# terminal, but Nix ensures that is is the case.  When doing this
+# manually, use setsid(1).)
+if ($sshKeys{$machine} =~ /^password:/) {
+    my $passwordFile = $sshKeys{$machine};
+    $passwordFile =~ s/^password://;
+    $sshOpts = "ssh -x";
+    $ENV{"SSH_PASSWORD_FILE"} = $passwordFile;
+    $ENV{"SSH_ASKPASS"} = "/tmp/writepass";
+
+    open WRITEPASS, ">/tmp/writepass" or die;
+    print WRITEPASS "#! /bin/sh\ncat \"\$SSH_PASSWORD_FILE\"";
+    close WRITEPASS;
+    chmod 0755, "/tmp/writepass" or die;
+}
+
+my $inputs = `cat inputs`; die if ($? != 0);
+$inputs =~ s/\n/ /g;
+
+my $outputs = `cat outputs`; die if ($? != 0);
+$outputs =~ s/\n/ /g;
+
+print "COPYING INPUTS...\n";
+
+my $maybeSign = "";
+$maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
+
+system("NIX_SSHOPTS=\"$sshOpts\" nix-copy-closure $machine $maybeSign $drvPath $inputs") == 0
+    or die "cannot copy inputs to $machine: $?";
+
+print "BUILDING...\n";
+
+system("ssh $sshOpts $machine 'nix-store -rvvK $drvPath'") == 0
+    or die "remote build on $machine failed: $?";
+
+print "REMOTE BUILD DONE: $drvPath on $machine\n";
+
+foreach my $output (split '\n', $outputs) {
+    my $maybeSignRemote = "";
+    $maybeSignRemote = "--sign" if $UID != 0;
+    
+    system("ssh $sshOpts $machine 'nix-store --export $maybeSignRemote $output' > dump") == 0
+	or die "cannot copy $output from $machine: $?";
+
+    # This doesn't work yet, since the caller has a lock on the output
+    # path.  We should move towards lock-free invocation of build
+    # hooks and substitutes.
+    #system("nix-store --import < dump") == 0
+    #    or die "cannot import $output: $?";
+
+    # Hack: skip the first 8 bytes (the nix-store --export next
+    # archive marker).  The archive follows.
+    system("(dd bs=1 count=8 of=/dev/null && cat) < dump | nix-store --restore $output") == 0
+	or die "cannot restore $output: $?";
+}

scripts/nix-build.in

@@ -77,9 +77,9 @@ EOF
         push @instArgs, ("--attr", $ARGV[$n]);
     }
 
-    elsif ($arg eq "--arg") {
+    elsif ($arg eq "--arg" || $arg eq "--argstr") {
-        die "$0: `--arg' requires two arguments\n" unless $n + 2 < scalar @ARGV;
+        die "$0: `$arg' requires two arguments\n" unless $n + 2 < scalar @ARGV;
-        push @instArgs, ("--arg", $ARGV[$n + 1], $ARGV[$n + 2]);
+        push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]);
         $n += 2;
     }
 

scripts/nix-prefetch-url.in

@@ -36,6 +36,28 @@ if test -n "$expHash"; then
 fi
 
 
+mkTempDir() {
+    local i=0
+    while true; do
+        if test -z "$TMPDIR"; then TMPDIR=/tmp; fi
+        tmpPath=$TMPDIR/nix-prefetch-url-$$-$i
+        if mkdir "$tmpPath"; then break; fi
+        # !!! to bad we can't check for ENOENT in mkdir, so this check
+        # is slightly racy (it bombs out if somebody just removed
+        # $tmpPath...).
+        if ! test -e "$tmpPath"; then exit 1; fi
+        i=$((i + 1))
+    done
+    trap removeTempDir EXIT SIGINT SIGQUIT
+}
+
+removeTempDir() {
+    if test -n "$tmpPath"; then
+        rm -rf "$tmpPath" || true
+    fi
+}
+
+
 doDownload() {
     @curl@ $cacheFlags --fail -# --location --max-redirs 20 --disable-epsv \
         --cookie-jar $tmpPath/cookies "$url" -o $tmpFile
@@ -46,9 +68,8 @@ doDownload() {
 # download the file and add it to the store.
 if test -z "$finalPath"; then
 
-    tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
+    mkTempDir
     tmpFile=$tmpPath/$name
-    mkdir $tmpPath # !!! retry if tmpPath already exists
 
     # Optionally do timestamp-based caching of the download.
     # Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is
@@ -98,8 +119,6 @@ if test -z "$finalPath"; then
         # Add the downloaded file to the Nix store.
         finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
 
-        if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
-
         if test -n "$expHash" -a "$expHash" != "$hash"; then
             echo "hash mismatch for URL \`$url'" >&2
             exit 1

scripts/nix-push.in

@@ -146,7 +146,7 @@ while (scalar @tmp > 0) {
     # probably wouldn't make that much sense; pumping lots of data
     # around just to compress them won't gain that much.
     $ENV{"NIX_BUILD_HOOK"} = "";
-    my $pid = open(READ, "$binDir/nix-store --realise @tmp2|")
+    my $pid = open(READ, "$binDir/nix-store --no-build-hook --realise @tmp2|")
         or die "cannot run nix-store";
     while (<READ>) {
         chomp;

src/libmain/shared.cc

@@ -223,6 +223,8 @@ static void initAndRun(int argc, char * * argv)
             readOnlyMode = true;
         else if (arg == "--max-silent-time")
             maxSilentTime = getIntArg(arg, i, args.end());
+        else if (arg == "--no-build-hook")
+            useBuildHook = false;
         else remaining.push_back(arg);
     }
 

src/libstore/build.cc

@@ -1282,6 +1282,7 @@ static string makeValidityRegistration(const PathSet & paths,
 
 DerivationGoal::HookReply DerivationGoal::tryBuildHook()
 {
+    if (!useBuildHook) return rpDecline;
     Path buildHook = getEnv("NIX_BUILD_HOOK");
     if (buildHook == "") return rpDecline;
     buildHook = absPath(buildHook);

src/libstore/gc.cc

@@ -26,6 +26,8 @@ static string gcLockName = "gc.lock";
 static string tempRootsDir = "temproots";
 static string gcRootsDir = "gcroots";
 
+const unsigned int defaultGcLevel = 1000;
+
 
 /* Acquire the global GC lock.  This is used to prevent new Nix
    processes from starting after the temporary root files have been
@@ -448,6 +450,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
         queryBoolSetting("gc-keep-outputs", false);
     bool gcKeepDerivations =
         queryBoolSetting("gc-keep-derivations", true);
+    unsigned int gcKeepOutputsThreshold = 
+        queryIntSetting ("gc-keep-outputs-threshold", defaultGcLevel);
 
 	//printMsg(lvlError, format("gcKeepOutputs %1% gcKeepDerivations: %2%") % gcKeepOutputs % gcKeepDerivations);
 
@@ -521,12 +525,30 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
         for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i)
             if (isDerivation(*i)) {
                 Derivation drv = derivationFromPathTxn(noTxn, *i);
+
+/*
+ * TODO REMOVE
+<<<<<<< .working
                 for (DerivationOutputs::iterator j = drv.outputs.begin();
                      j != drv.outputs.end(); ++j)
                     if (store->isValidPath(j->second.path))
                         computeFSClosure(j->second.path, livePaths, true, true, 0);
                     else if (store->isValidStatePath(j->second.path))
                     	computeFSClosure(j->second.path, livePaths, true, true, 0);
+=======
+*/
+
+			string gcLevelStr = drv.env["__gcLevel"];
+			int gcLevel;
+			if (!string2Int(gcLevelStr,gcLevel)) {
+			    gcLevel = defaultGcLevel;
+			}
+			
+			if (gcLevel >= gcKeepOutputsThreshold)    
+			    for (DerivationOutputs::iterator j = drv.outputs.begin();
+			            j != drv.outputs.end(); ++j)
+				if (store->isValidPath(j->second.path) || store->isValidStatePath(j->second.path))
+				    computeFSClosure(j->second.path, livePaths, true, true, 0);
            }
     }
 

src/libstore/globals.cc

@@ -29,6 +29,7 @@ bool readOnlyMode = false;
 string thisSystem = "unset";
 unsigned int maxSilentTime = 0;
 Paths substituters;
+bool useBuildHook = true;
 
 
 static bool settingsRead = false;

src/libstore/globals.hh

@@ -80,6 +80,10 @@ extern unsigned int maxSilentTime;
    from a CD. */
 extern Paths substituters;
 
+/* Whether to use build hooks (for distributed builds).  Sometimes
+   users want to disable this from the command-line. */
+extern bool useBuildHook;
+
 Strings querySetting(const string & name, const Strings & def);
 
 string querySetting(const string & name, const string & def);

src/libstore/remote-store.cc

@@ -93,7 +93,7 @@ RemoteStore::RemoteStore()
         unsigned int magic = readInt(from);
         if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
 
-        unsigned int daemonVersion = readInt(from);
+        daemonVersion = readInt(from);
         if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
             throw Error("Nix daemon protocol version not supported");
         writeInt(PROTOCOL_VERSION, to);
@@ -203,6 +203,8 @@ void RemoteStore::setOptions()
     writeInt(verbosity, to);
     writeInt(maxBuildJobs, to);
     writeInt(maxSilentTime, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
+        writeInt(useBuildHook, to);
     processStderr();
 }
 
@@ -312,7 +314,9 @@ Path RemoteStore::queryDeriver(const Path & path)
     writeInt(wopQueryDeriver, to);
     writeString(path, to);
     processStderr();
-    return readStorePath(from);
+    Path drvPath = readString(from);
+    if (drvPath != "") assertStorePath(drvPath);
+    return drvPath;
 }
 
 

src/libstore/remote-store.hh

@@ -117,6 +117,7 @@ private:
     FdSink to;
     FdSource from;
     Pid child;
+    unsigned int daemonVersion;
 
     void processStderr(Sink * sink = 0, Source * source = 0);
 

src/libstore/worker-protocol.hh

@@ -8,8 +8,9 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x101
+#define PROTOCOL_VERSION 0x102
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
+#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
 typedef enum {

src/nix-store/nix-store.cc

@@ -588,12 +588,17 @@ static void opExport(Strings opFlags, Strings opArgs)
 
 static void opImport(Strings opFlags, Strings opArgs)
 {
-    if (!opFlags.empty()) throw UsageError("unknown flag");
+    bool requireSignature = false;
+    for (Strings::iterator i = opFlags.begin();
+         i != opFlags.end(); ++i)
+        if (*i == "--require-signature") requireSignature = true;
+        else throw UsageError(format("unknown flag `%1%'") % *i);
+    
     if (!opArgs.empty()) throw UsageError("no arguments expected");
     
     FdSource source(STDIN_FILENO);
     while (readInt(source) == 1)
-        cout << format("%1%\n") % store->importPath(false, source) << std::flush;
+        cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush;
 }
 
 

src/nix-worker/nix-worker.cc

@@ -225,7 +225,8 @@ struct TunnelSource : Source
 };
 
 
-static void performOp(Source & from, Sink & to, unsigned int op)
+static void performOp(unsigned int clientVersion,
+    Source & from, Sink & to, unsigned int op)
 {
     switch (op) {
 
@@ -666,6 +667,8 @@ static void performOp(Source & from, Sink & to, unsigned int op)
         verbosity = (Verbosity) readInt(from);
         maxBuildJobs = readInt(from);
         maxSilentTime = readInt(from);
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
+            useBuildHook = readInt(from) != 0;
         startWork();
         stopWork();
         break;
@@ -755,7 +758,7 @@ static void processConnection()
 
         try {
         	printMsg(lvlInfo, format("Processing op '%1%' with pid '%2%'") % op % myPid);
-            performOp(from, to, op);
+            performOp(clientVersion, from, to, op);
             printMsg(lvlInfo, format("Processed op '%1%'") % op);
         } catch (Error & e) {
             stopWork(false, e.msg());