From 55b07d65b1176582b6fa3ebc3a0455a7788d2565 Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Mon, 19 Nov 2007 11:47:41 +0000 Subject: [PATCH] Merged trunk R9751 back in. --- mergeTrunkBackIn.sh | 4 +- scripts/Makefile.am | 7 +- scripts/build-remote.pl.in | 208 ++++++++++++++++++++++++++++++++ scripts/nix-build.in | 6 +- scripts/nix-prefetch-url.in | 27 ++++- scripts/nix-push.in | 2 +- src/libmain/shared.cc | 2 + src/libstore/build.cc | 1 + src/libstore/gc.cc | 24 +++- src/libstore/globals.cc | 1 + src/libstore/globals.hh | 4 + src/libstore/remote-store.cc | 8 +- src/libstore/remote-store.hh | 1 + src/libstore/worker-protocol.hh | 3 +- src/nix-store/nix-store.cc | 9 +- src/nix-worker/nix-worker.cc | 7 +- 16 files changed, 294 insertions(+), 20 deletions(-) create mode 100755 scripts/build-remote.pl.in diff --git a/mergeTrunkBackIn.sh b/mergeTrunkBackIn.sh index 0fb0b1b78..9b400ea7d 100755 --- a/mergeTrunkBackIn.sh +++ b/mergeTrunkBackIn.sh @@ -1,4 +1,4 @@ -svn merge -r 9561:9584 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk +svn merge -r 9584:9751 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk #already done: # 8628 @@ -27,4 +27,4 @@ svn merge -r 9561:9584 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk # 9549 # 9561 # 9584 - +# 9751 diff --git a/scripts/Makefile.am b/scripts/Makefile.am index e16b36d61..e79db31b7 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -4,7 +4,8 @@ bin_SCRIPTS = nix-collect-garbage \ nix-pack-closure nix-unpack-closure \ nix-copy-closure -noinst_SCRIPTS = nix-profile.sh generate-patches.pl find-runtime-roots.pl +noinst_SCRIPTS = nix-profile.sh generate-patches.pl \ + find-runtime-roots.pl build-remote.pl nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl @@ -17,6 +18,7 @@ install-exec-local: readmanifest.pm download-using-manifests.pl find-runtime-roo $(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix + $(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix include ../substitute.mk @@ -32,4 +34,5 @@ EXTRA_DIST = nix-collect-garbage.in \ generate-patches.pl.in \ nix-pack-closure.in nix-unpack-closure.in \ nix-copy-closure.in \ - find-runtime-roots.pl.in + find-runtime-roots.pl.in \ + build-remote.pl.in diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in new file mode 100755 index 000000000..9d582c566 --- /dev/null +++ b/scripts/build-remote.pl.in @@ -0,0 +1,208 @@ +#! @perl@ -w + +use strict; +use Fcntl ':flock'; +use English '-no_match_vars'; + +# General operation: +# +# Try to find a free machine of type $neededSystem. We do this as +# follows: +# - We acquire an exclusive lock on $currentLoad/main-lock. +# - For each machine $machine of type $neededSystem and for each $slot +# less than the maximum load for that machine, we try to get an +# exclusive lock on $currentLoad/$machine-$slot (without blocking). +# If we get such a lock, we send "accept" to the caller. Otherwise, +# we send "postpone" and exit. +# - We release the exclusive lock on $currentLoad/main-lock. +# - We perform the build on $neededSystem. +# - We release the exclusive lock on $currentLoad/$machine-$slot. +# +# The nice thing about this scheme is that if we die prematurely, the +# locks are released automatically. + +my $loadIncreased = 0; + +my $amWilling = shift @ARGV; +my $localSystem = shift @ARGV; +my $neededSystem = shift @ARGV; +my $drvPath = shift @ARGV; + +sub sendReply { + my $reply = shift; + open OUT, ">&3" or die; + print OUT "$reply\n"; + close OUT; +} + +sub decline { + sendReply "decline"; + exit 0; +} + +my $currentLoad = $ENV{"NIX_CURRENT_LOAD"}; +decline unless defined $currentLoad; +mkdir $currentLoad, 0777 or die unless -d $currentLoad; + +my $conf = $ENV{"NIX_REMOTE_SYSTEMS"}; +decline if !defined $conf || ! -e $conf; + +# Decline if the local system can do the build. +decline if $amWilling && ($localSystem eq $neededSystem); + + +# Otherwise find a willing remote machine. +my %machines; +my %systemTypes; +my %sshKeys; +my %maxJobs; +my %curJobs; + + +# Read the list of machines. +open CONF, "< $conf" or die; + +while () { + chomp; + s/\#.*$//g; + next if /^\s*$/; + /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die; + $machines{$1} = ""; + $systemTypes{$1} = $2; + $sshKeys{$1} = $3; + $maxJobs{$1} = $4; +} + +close CONF; + + +# Acquire the exclusive lock on $currentLoad/main-lock. +my $mainLock = "$currentLoad/main-lock"; +open MAINLOCK, ">>$mainLock" or die; +flock(MAINLOCK, LOCK_EX) or die; + + +# Find a suitable system. +my $rightType = 0; +my $machine; +LOOP: foreach my $cur (keys %machines) { + if ($neededSystem eq $systemTypes{$cur}) { + $rightType = 1; + + # We have a machine of the right type. Try to get a lock on + # one of the machine's lock files. + my $slot = 0; + while ($slot < $maxJobs{$cur}) { + my $slotLock = "$currentLoad/$cur-$slot"; + open SLOTLOCK, ">>$slotLock" or die; + if (flock(SLOTLOCK, LOCK_EX | LOCK_NB)) { + $machine = $cur; + last LOOP; + } + close SLOTLOCK; + $slot++; + } + } +} + +close MAINLOCK; + + +# Didn't find one? +if (!defined $machine) { + if ($rightType) { + sendReply "postpone"; + exit 0; + } else { + decline; + } +} + +# Yes we did, accept. +sendReply "accept"; +open IN, "<&4" or die; +my $x = ; +chomp $x; +#print "got $x\n"; +close IN; + +if ($x ne "okay") { + exit 0; +} + + +# Do the actual job. +print "BUILDING REMOTE: $drvPath on $machine\n"; + +# Make sure that we don't get any SSH passphrase or host key popups - +# if there is any problem it should fail, not do something +# interactive. +$ENV{"DISPLAY"} = ""; +$ENV{"SSH_PASSWORD_FILE="} = ""; +$ENV{"SSH_ASKPASS="} = ""; + +my $sshOpts = "-i $sshKeys{$machine} -x"; + +# Hack to support Cygwin: if we login without a password, we don't +# have exactly the same right as when we do. This causes the +# Microsoft C compiler to fail with certain flags: +# +# http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676 +# +# So as a workaround, we pass a verbatim password. ssh tries to makes +# this very hard; the trick is to make it call SSH_ASKPASS to get the +# password. (It only calls this command when there is no controlling +# terminal, but Nix ensures that is is the case. When doing this +# manually, use setsid(1).) +if ($sshKeys{$machine} =~ /^password:/) { + my $passwordFile = $sshKeys{$machine}; + $passwordFile =~ s/^password://; + $sshOpts = "ssh -x"; + $ENV{"SSH_PASSWORD_FILE"} = $passwordFile; + $ENV{"SSH_ASKPASS"} = "/tmp/writepass"; + + open WRITEPASS, ">/tmp/writepass" or die; + print WRITEPASS "#! /bin/sh\ncat \"\$SSH_PASSWORD_FILE\""; + close WRITEPASS; + chmod 0755, "/tmp/writepass" or die; +} + +my $inputs = `cat inputs`; die if ($? != 0); +$inputs =~ s/\n/ /g; + +my $outputs = `cat outputs`; die if ($? != 0); +$outputs =~ s/\n/ /g; + +print "COPYING INPUTS...\n"; + +my $maybeSign = ""; +$maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec"; + +system("NIX_SSHOPTS=\"$sshOpts\" nix-copy-closure $machine $maybeSign $drvPath $inputs") == 0 + or die "cannot copy inputs to $machine: $?"; + +print "BUILDING...\n"; + +system("ssh $sshOpts $machine 'nix-store -rvvK $drvPath'") == 0 + or die "remote build on $machine failed: $?"; + +print "REMOTE BUILD DONE: $drvPath on $machine\n"; + +foreach my $output (split '\n', $outputs) { + my $maybeSignRemote = ""; + $maybeSignRemote = "--sign" if $UID != 0; + + system("ssh $sshOpts $machine 'nix-store --export $maybeSignRemote $output' > dump") == 0 + or die "cannot copy $output from $machine: $?"; + + # This doesn't work yet, since the caller has a lock on the output + # path. We should move towards lock-free invocation of build + # hooks and substitutes. + #system("nix-store --import < dump") == 0 + # or die "cannot import $output: $?"; + + # Hack: skip the first 8 bytes (the nix-store --export next + # archive marker). The archive follows. + system("(dd bs=1 count=8 of=/dev/null && cat) < dump | nix-store --restore $output") == 0 + or die "cannot restore $output: $?"; +} diff --git a/scripts/nix-build.in b/scripts/nix-build.in index bce4a8007..1d2584bfb 100644 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -77,9 +77,9 @@ EOF push @instArgs, ("--attr", $ARGV[$n]); } - elsif ($arg eq "--arg") { - die "$0: `--arg' requires two arguments\n" unless $n + 2 < scalar @ARGV; - push @instArgs, ("--arg", $ARGV[$n + 1], $ARGV[$n + 2]); + elsif ($arg eq "--arg" || $arg eq "--argstr") { + die "$0: `$arg' requires two arguments\n" unless $n + 2 < scalar @ARGV; + push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]); $n += 2; } diff --git a/scripts/nix-prefetch-url.in b/scripts/nix-prefetch-url.in index bd970e373..ad4cfa30b 100644 --- a/scripts/nix-prefetch-url.in +++ b/scripts/nix-prefetch-url.in @@ -36,6 +36,28 @@ if test -n "$expHash"; then fi +mkTempDir() { + local i=0 + while true; do + if test -z "$TMPDIR"; then TMPDIR=/tmp; fi + tmpPath=$TMPDIR/nix-prefetch-url-$$-$i + if mkdir "$tmpPath"; then break; fi + # !!! to bad we can't check for ENOENT in mkdir, so this check + # is slightly racy (it bombs out if somebody just removed + # $tmpPath...). + if ! test -e "$tmpPath"; then exit 1; fi + i=$((i + 1)) + done + trap removeTempDir EXIT SIGINT SIGQUIT +} + +removeTempDir() { + if test -n "$tmpPath"; then + rm -rf "$tmpPath" || true + fi +} + + doDownload() { @curl@ $cacheFlags --fail -# --location --max-redirs 20 --disable-epsv \ --cookie-jar $tmpPath/cookies "$url" -o $tmpFile @@ -46,9 +68,8 @@ doDownload() { # download the file and add it to the store. if test -z "$finalPath"; then - tmpPath=/tmp/nix-prefetch-url-$$ # !!! security? + mkTempDir tmpFile=$tmpPath/$name - mkdir $tmpPath # !!! retry if tmpPath already exists # Optionally do timestamp-based caching of the download. # Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is @@ -98,8 +119,6 @@ if test -z "$finalPath"; then # Add the downloaded file to the Nix store. finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile) - if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi - if test -n "$expHash" -a "$expHash" != "$hash"; then echo "hash mismatch for URL \`$url'" >&2 exit 1 diff --git a/scripts/nix-push.in b/scripts/nix-push.in index dd00841d7..6250f8b0a 100644 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -146,7 +146,7 @@ while (scalar @tmp > 0) { # probably wouldn't make that much sense; pumping lots of data # around just to compress them won't gain that much. $ENV{"NIX_BUILD_HOOK"} = ""; - my $pid = open(READ, "$binDir/nix-store --realise @tmp2|") + my $pid = open(READ, "$binDir/nix-store --no-build-hook --realise @tmp2|") or die "cannot run nix-store"; while () { chomp; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 472e38ae7..e6a4fd968 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -223,6 +223,8 @@ static void initAndRun(int argc, char * * argv) readOnlyMode = true; else if (arg == "--max-silent-time") maxSilentTime = getIntArg(arg, i, args.end()); + else if (arg == "--no-build-hook") + useBuildHook = false; else remaining.push_back(arg); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2c89465b7..1b505d67b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1282,6 +1282,7 @@ static string makeValidityRegistration(const PathSet & paths, DerivationGoal::HookReply DerivationGoal::tryBuildHook() { + if (!useBuildHook) return rpDecline; Path buildHook = getEnv("NIX_BUILD_HOOK"); if (buildHook == "") return rpDecline; buildHook = absPath(buildHook); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 07a78f0a7..0be0eda83 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -26,6 +26,8 @@ static string gcLockName = "gc.lock"; static string tempRootsDir = "temproots"; static string gcRootsDir = "gcroots"; +const unsigned int defaultGcLevel = 1000; + /* Acquire the global GC lock. This is used to prevent new Nix processes from starting after the temporary root files have been @@ -448,6 +450,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, queryBoolSetting("gc-keep-outputs", false); bool gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true); + unsigned int gcKeepOutputsThreshold = + queryIntSetting ("gc-keep-outputs-threshold", defaultGcLevel); //printMsg(lvlError, format("gcKeepOutputs %1% gcKeepDerivations: %2%") % gcKeepOutputs % gcKeepDerivations); @@ -521,13 +525,31 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i) if (isDerivation(*i)) { Derivation drv = derivationFromPathTxn(noTxn, *i); + +/* + * TODO REMOVE +<<<<<<< .working for (DerivationOutputs::iterator j = drv.outputs.begin(); j != drv.outputs.end(); ++j) if (store->isValidPath(j->second.path)) computeFSClosure(j->second.path, livePaths, true, true, 0); else if (store->isValidStatePath(j->second.path)) computeFSClosure(j->second.path, livePaths, true, true, 0); - } +======= +*/ + + string gcLevelStr = drv.env["__gcLevel"]; + int gcLevel; + if (!string2Int(gcLevelStr,gcLevel)) { + gcLevel = defaultGcLevel; + } + + if (gcLevel >= gcKeepOutputsThreshold) + for (DerivationOutputs::iterator j = drv.outputs.begin(); + j != drv.outputs.end(); ++j) + if (store->isValidPath(j->second.path) || store->isValidStatePath(j->second.path)) + computeFSClosure(j->second.path, livePaths, true, true, 0); + } } if (action == gcReturnLive) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 18bd17c43..d9c636303 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -29,6 +29,7 @@ bool readOnlyMode = false; string thisSystem = "unset"; unsigned int maxSilentTime = 0; Paths substituters; +bool useBuildHook = true; static bool settingsRead = false; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 252a8cf38..cc8724f65 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -80,6 +80,10 @@ extern unsigned int maxSilentTime; from a CD. */ extern Paths substituters; +/* Whether to use build hooks (for distributed builds). Sometimes + users want to disable this from the command-line. */ +extern bool useBuildHook; + Strings querySetting(const string & name, const Strings & def); string querySetting(const string & name, const string & def); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c0c56222e..e3e4e4e0b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -93,7 +93,7 @@ RemoteStore::RemoteStore() unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - unsigned int daemonVersion = readInt(from); + daemonVersion = readInt(from); if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) throw Error("Nix daemon protocol version not supported"); writeInt(PROTOCOL_VERSION, to); @@ -203,6 +203,8 @@ void RemoteStore::setOptions() writeInt(verbosity, to); writeInt(maxBuildJobs, to); writeInt(maxSilentTime, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) + writeInt(useBuildHook, to); processStderr(); } @@ -312,7 +314,9 @@ Path RemoteStore::queryDeriver(const Path & path) writeInt(wopQueryDeriver, to); writeString(path, to); processStderr(); - return readStorePath(from); + Path drvPath = readString(from); + if (drvPath != "") assertStorePath(drvPath); + return drvPath; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9290d00bd..827211b9f 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -117,6 +117,7 @@ private: FdSink to; FdSource from; Pid child; + unsigned int daemonVersion; void processStderr(Sink * sink = 0, Source * source = 0); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 31d56bb54..60e4a0339 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,8 +8,9 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x101 +#define PROTOCOL_VERSION 0x102 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) typedef enum { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 41fa00a2d..d7d046d75 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -588,12 +588,17 @@ static void opExport(Strings opFlags, Strings opArgs) static void opImport(Strings opFlags, Strings opArgs) { - if (!opFlags.empty()) throw UsageError("unknown flag"); + bool requireSignature = false; + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--require-signature") requireSignature = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); while (readInt(source) == 1) - cout << format("%1%\n") % store->importPath(false, source) << std::flush; + cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush; } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index b2962b5e7..5ee942576 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -225,7 +225,8 @@ struct TunnelSource : Source }; -static void performOp(Source & from, Sink & to, unsigned int op) +static void performOp(unsigned int clientVersion, + Source & from, Sink & to, unsigned int op) { switch (op) { @@ -666,6 +667,8 @@ static void performOp(Source & from, Sink & to, unsigned int op) verbosity = (Verbosity) readInt(from); maxBuildJobs = readInt(from); maxSilentTime = readInt(from); + if (GET_PROTOCOL_MINOR(clientVersion) >= 2) + useBuildHook = readInt(from) != 0; startWork(); stopWork(); break; @@ -755,7 +758,7 @@ static void processConnection() try { printMsg(lvlInfo, format("Processing op '%1%' with pid '%2%'") % op % myPid); - performOp(from, to, op); + performOp(clientVersion, from, to, op); printMsg(lvlInfo, format("Processed op '%1%'") % op); } catch (Error & e) { stopWork(false, e.msg());