mirror of
https://github.com/NixOS/nix.git
synced 2025-11-26 04:00:59 +01:00
Merged trunk R9751 back in.
This commit is contained in:
Wouter den Breejen
parent
4e11da960c
commit
55b07d65b1
16 changed files with 294 additions and 20 deletions
|
|
@ -1,4 +1,4 @@
|
|||
svn merge -r 9561:9584 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk
|
||||
svn merge -r 9584:9751 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk
|
||||
|
||||
#already done:
|
||||
# 8628
|
||||
|
|
@ -27,4 +27,4 @@ svn merge -r 9561:9584 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk
|
|||
# 9549
|
||||
# 9561
|
||||
# 9584
|
||||
|
||||
# 9751
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ bin_SCRIPTS = nix-collect-garbage \
|
|||
nix-pack-closure nix-unpack-closure \
|
||||
nix-copy-closure
|
||||
|
||||
noinst_SCRIPTS = nix-profile.sh generate-patches.pl find-runtime-roots.pl
|
||||
noinst_SCRIPTS = nix-profile.sh generate-patches.pl \
|
||||
find-runtime-roots.pl build-remote.pl
|
||||
|
||||
nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl
|
||||
|
||||
|
|
@ -17,6 +18,7 @@ install-exec-local: readmanifest.pm download-using-manifests.pl find-runtime-roo
|
|||
$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix
|
||||
$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix
|
||||
$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix
|
||||
$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix
|
||||
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
|
||||
|
||||
include ../substitute.mk
|
||||
|
|
@ -32,4 +34,5 @@ EXTRA_DIST = nix-collect-garbage.in \
|
|||
generate-patches.pl.in \
|
||||
nix-pack-closure.in nix-unpack-closure.in \
|
||||
nix-copy-closure.in \
|
||||
find-runtime-roots.pl.in
|
||||
find-runtime-roots.pl.in \
|
||||
build-remote.pl.in
|
||||
|
|
|
|||
208
scripts/build-remote.pl.in
Executable file
208
scripts/build-remote.pl.in
Executable file
|
|
@ -0,0 +1,208 @@
|
|||
#! @perl@ -w
|
||||
|
||||
use strict;
|
||||
use Fcntl ':flock';
|
||||
use English '-no_match_vars';
|
||||
|
||||
# General operation:
|
||||
#
|
||||
# Try to find a free machine of type $neededSystem. We do this as
|
||||
# follows:
|
||||
# - We acquire an exclusive lock on $currentLoad/main-lock.
|
||||
# - For each machine $machine of type $neededSystem and for each $slot
|
||||
# less than the maximum load for that machine, we try to get an
|
||||
# exclusive lock on $currentLoad/$machine-$slot (without blocking).
|
||||
# If we get such a lock, we send "accept" to the caller. Otherwise,
|
||||
# we send "postpone" and exit.
|
||||
# - We release the exclusive lock on $currentLoad/main-lock.
|
||||
# - We perform the build on $neededSystem.
|
||||
# - We release the exclusive lock on $currentLoad/$machine-$slot.
|
||||
#
|
||||
# The nice thing about this scheme is that if we die prematurely, the
|
||||
# locks are released automatically.
|
||||
|
||||
my $loadIncreased = 0;
|
||||
|
||||
my $amWilling = shift @ARGV;
|
||||
my $localSystem = shift @ARGV;
|
||||
my $neededSystem = shift @ARGV;
|
||||
my $drvPath = shift @ARGV;
|
||||
|
||||
sub sendReply {
|
||||
my $reply = shift;
|
||||
open OUT, ">&3" or die;
|
||||
print OUT "$reply\n";
|
||||
close OUT;
|
||||
}
|
||||
|
||||
sub decline {
|
||||
sendReply "decline";
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my $currentLoad = $ENV{"NIX_CURRENT_LOAD"};
|
||||
decline unless defined $currentLoad;
|
||||
mkdir $currentLoad, 0777 or die unless -d $currentLoad;
|
||||
|
||||
my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
|
||||
decline if !defined $conf || ! -e $conf;
|
||||
|
||||
# Decline if the local system can do the build.
|
||||
decline if $amWilling && ($localSystem eq $neededSystem);
|
||||
|
||||
|
||||
# Otherwise find a willing remote machine.
|
||||
my %machines;
|
||||
my %systemTypes;
|
||||
my %sshKeys;
|
||||
my %maxJobs;
|
||||
my %curJobs;
|
||||
|
||||
|
||||
# Read the list of machines.
|
||||
open CONF, "< $conf" or die;
|
||||
|
||||
while (<CONF>) {
|
||||
chomp;
|
||||
s/\#.*$//g;
|
||||
next if /^\s*$/;
|
||||
/^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die;
|
||||
$machines{$1} = "";
|
||||
$systemTypes{$1} = $2;
|
||||
$sshKeys{$1} = $3;
|
||||
$maxJobs{$1} = $4;
|
||||
}
|
||||
|
||||
close CONF;
|
||||
|
||||
|
||||
# Acquire the exclusive lock on $currentLoad/main-lock.
|
||||
my $mainLock = "$currentLoad/main-lock";
|
||||
open MAINLOCK, ">>$mainLock" or die;
|
||||
flock(MAINLOCK, LOCK_EX) or die;
|
||||
|
||||
|
||||
# Find a suitable system.
|
||||
my $rightType = 0;
|
||||
my $machine;
|
||||
LOOP: foreach my $cur (keys %machines) {
|
||||
if ($neededSystem eq $systemTypes{$cur}) {
|
||||
$rightType = 1;
|
||||
|
||||
# We have a machine of the right type. Try to get a lock on
|
||||
# one of the machine's lock files.
|
||||
my $slot = 0;
|
||||
while ($slot < $maxJobs{$cur}) {
|
||||
my $slotLock = "$currentLoad/$cur-$slot";
|
||||
open SLOTLOCK, ">>$slotLock" or die;
|
||||
if (flock(SLOTLOCK, LOCK_EX | LOCK_NB)) {
|
||||
$machine = $cur;
|
||||
last LOOP;
|
||||
}
|
||||
close SLOTLOCK;
|
||||
$slot++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
close MAINLOCK;
|
||||
|
||||
|
||||
# Didn't find one?
|
||||
if (!defined $machine) {
|
||||
if ($rightType) {
|
||||
sendReply "postpone";
|
||||
exit 0;
|
||||
} else {
|
||||
decline;
|
||||
}
|
||||
}
|
||||
|
||||
# Yes we did, accept.
|
||||
sendReply "accept";
|
||||
open IN, "<&4" or die;
|
||||
my $x = <IN>;
|
||||
chomp $x;
|
||||
#print "got $x\n";
|
||||
close IN;
|
||||
|
||||
if ($x ne "okay") {
|
||||
exit 0;
|
||||
}
|
||||
|
||||
|
||||
# Do the actual job.
|
||||
print "BUILDING REMOTE: $drvPath on $machine\n";
|
||||
|
||||
# Make sure that we don't get any SSH passphrase or host key popups -
|
||||
# if there is any problem it should fail, not do something
|
||||
# interactive.
|
||||
$ENV{"DISPLAY"} = "";
|
||||
$ENV{"SSH_PASSWORD_FILE="} = "";
|
||||
$ENV{"SSH_ASKPASS="} = "";
|
||||
|
||||
my $sshOpts = "-i $sshKeys{$machine} -x";
|
||||
|
||||
# Hack to support Cygwin: if we login without a password, we don't
|
||||
# have exactly the same right as when we do. This causes the
|
||||
# Microsoft C compiler to fail with certain flags:
|
||||
#
|
||||
# http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676
|
||||
#
|
||||
# So as a workaround, we pass a verbatim password. ssh tries to makes
|
||||
# this very hard; the trick is to make it call SSH_ASKPASS to get the
|
||||
# password. (It only calls this command when there is no controlling
|
||||
# terminal, but Nix ensures that is is the case. When doing this
|
||||
# manually, use setsid(1).)
|
||||
if ($sshKeys{$machine} =~ /^password:/) {
|
||||
my $passwordFile = $sshKeys{$machine};
|
||||
$passwordFile =~ s/^password://;
|
||||
$sshOpts = "ssh -x";
|
||||
$ENV{"SSH_PASSWORD_FILE"} = $passwordFile;
|
||||
$ENV{"SSH_ASKPASS"} = "/tmp/writepass";
|
||||
|
||||
open WRITEPASS, ">/tmp/writepass" or die;
|
||||
print WRITEPASS "#! /bin/sh\ncat \"\$SSH_PASSWORD_FILE\"";
|
||||
close WRITEPASS;
|
||||
chmod 0755, "/tmp/writepass" or die;
|
||||
}
|
||||
|
||||
my $inputs = `cat inputs`; die if ($? != 0);
|
||||
$inputs =~ s/\n/ /g;
|
||||
|
||||
my $outputs = `cat outputs`; die if ($? != 0);
|
||||
$outputs =~ s/\n/ /g;
|
||||
|
||||
print "COPYING INPUTS...\n";
|
||||
|
||||
my $maybeSign = "";
|
||||
$maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
|
||||
|
||||
system("NIX_SSHOPTS=\"$sshOpts\" nix-copy-closure $machine $maybeSign $drvPath $inputs") == 0
|
||||
or die "cannot copy inputs to $machine: $?";
|
||||
|
||||
print "BUILDING...\n";
|
||||
|
||||
system("ssh $sshOpts $machine 'nix-store -rvvK $drvPath'") == 0
|
||||
or die "remote build on $machine failed: $?";
|
||||
|
||||
print "REMOTE BUILD DONE: $drvPath on $machine\n";
|
||||
|
||||
foreach my $output (split '\n', $outputs) {
|
||||
my $maybeSignRemote = "";
|
||||
$maybeSignRemote = "--sign" if $UID != 0;
|
||||
|
||||
system("ssh $sshOpts $machine 'nix-store --export $maybeSignRemote $output' > dump") == 0
|
||||
or die "cannot copy $output from $machine: $?";
|
||||
|
||||
# This doesn't work yet, since the caller has a lock on the output
|
||||
# path. We should move towards lock-free invocation of build
|
||||
# hooks and substitutes.
|
||||
#system("nix-store --import < dump") == 0
|
||||
# or die "cannot import $output: $?";
|
||||
|
||||
# Hack: skip the first 8 bytes (the nix-store --export next
|
||||
# archive marker). The archive follows.
|
||||
system("(dd bs=1 count=8 of=/dev/null && cat) < dump | nix-store --restore $output") == 0
|
||||
or die "cannot restore $output: $?";
|
||||
}
|
||||
|
|
@ -77,9 +77,9 @@ EOF
|
|||
push @instArgs, ("--attr", $ARGV[$n]);
|
||||
}
|
||||
|
||||
elsif ($arg eq "--arg") {
|
||||
die "$0: `--arg' requires two arguments\n" unless $n + 2 < scalar @ARGV;
|
||||
push @instArgs, ("--arg", $ARGV[$n + 1], $ARGV[$n + 2]);
|
||||
elsif ($arg eq "--arg" || $arg eq "--argstr") {
|
||||
die "$0: `$arg' requires two arguments\n" unless $n + 2 < scalar @ARGV;
|
||||
push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]);
|
||||
$n += 2;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,28 @@ if test -n "$expHash"; then
|
|||
fi
|
||||
|
||||
|
||||
mkTempDir() {
|
||||
local i=0
|
||||
while true; do
|
||||
if test -z "$TMPDIR"; then TMPDIR=/tmp; fi
|
||||
tmpPath=$TMPDIR/nix-prefetch-url-$$-$i
|
||||
if mkdir "$tmpPath"; then break; fi
|
||||
# !!! to bad we can't check for ENOENT in mkdir, so this check
|
||||
# is slightly racy (it bombs out if somebody just removed
|
||||
# $tmpPath...).
|
||||
if ! test -e "$tmpPath"; then exit 1; fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
trap removeTempDir EXIT SIGINT SIGQUIT
|
||||
}
|
||||
|
||||
removeTempDir() {
|
||||
if test -n "$tmpPath"; then
|
||||
rm -rf "$tmpPath" || true
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
doDownload() {
|
||||
@curl@ $cacheFlags --fail -# --location --max-redirs 20 --disable-epsv \
|
||||
--cookie-jar $tmpPath/cookies "$url" -o $tmpFile
|
||||
|
|
@ -46,9 +68,8 @@ doDownload() {
|
|||
# download the file and add it to the store.
|
||||
if test -z "$finalPath"; then
|
||||
|
||||
tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
|
||||
mkTempDir
|
||||
tmpFile=$tmpPath/$name
|
||||
mkdir $tmpPath # !!! retry if tmpPath already exists
|
||||
|
||||
# Optionally do timestamp-based caching of the download.
|
||||
# Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is
|
||||
|
|
@ -98,8 +119,6 @@ if test -z "$finalPath"; then
|
|||
# Add the downloaded file to the Nix store.
|
||||
finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
|
||||
|
||||
if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
|
||||
|
||||
if test -n "$expHash" -a "$expHash" != "$hash"; then
|
||||
echo "hash mismatch for URL \`$url'" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@ while (scalar @tmp > 0) {
|
|||
# probably wouldn't make that much sense; pumping lots of data
|
||||
# around just to compress them won't gain that much.
|
||||
$ENV{"NIX_BUILD_HOOK"} = "";
|
||||
my $pid = open(READ, "$binDir/nix-store --realise @tmp2|")
|
||||
my $pid = open(READ, "$binDir/nix-store --no-build-hook --realise @tmp2|")
|
||||
or die "cannot run nix-store";
|
||||
while (<READ>) {
|
||||
chomp;
|
||||
|
|
|
|||
|
|
@ -223,6 +223,8 @@ static void initAndRun(int argc, char * * argv)
|
|||
readOnlyMode = true;
|
||||
else if (arg == "--max-silent-time")
|
||||
maxSilentTime = getIntArg(arg, i, args.end());
|
||||
else if (arg == "--no-build-hook")
|
||||
useBuildHook = false;
|
||||
else remaining.push_back(arg);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1282,6 +1282,7 @@ static string makeValidityRegistration(const PathSet & paths,
|
|||
|
||||
DerivationGoal::HookReply DerivationGoal::tryBuildHook()
|
||||
{
|
||||
if (!useBuildHook) return rpDecline;
|
||||
Path buildHook = getEnv("NIX_BUILD_HOOK");
|
||||
if (buildHook == "") return rpDecline;
|
||||
buildHook = absPath(buildHook);
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ static string gcLockName = "gc.lock";
|
|||
static string tempRootsDir = "temproots";
|
||||
static string gcRootsDir = "gcroots";
|
||||
|
||||
const unsigned int defaultGcLevel = 1000;
|
||||
|
||||
|
||||
/* Acquire the global GC lock. This is used to prevent new Nix
|
||||
processes from starting after the temporary root files have been
|
||||
|
|
@ -448,6 +450,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
queryBoolSetting("gc-keep-outputs", false);
|
||||
bool gcKeepDerivations =
|
||||
queryBoolSetting("gc-keep-derivations", true);
|
||||
unsigned int gcKeepOutputsThreshold =
|
||||
queryIntSetting ("gc-keep-outputs-threshold", defaultGcLevel);
|
||||
|
||||
//printMsg(lvlError, format("gcKeepOutputs %1% gcKeepDerivations: %2%") % gcKeepOutputs % gcKeepDerivations);
|
||||
|
||||
|
|
@ -521,12 +525,30 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i)
|
||||
if (isDerivation(*i)) {
|
||||
Derivation drv = derivationFromPathTxn(noTxn, *i);
|
||||
|
||||
/*
|
||||
* TODO REMOVE
|
||||
<<<<<<< .working
|
||||
for (DerivationOutputs::iterator j = drv.outputs.begin();
|
||||
j != drv.outputs.end(); ++j)
|
||||
if (store->isValidPath(j->second.path))
|
||||
computeFSClosure(j->second.path, livePaths, true, true, 0);
|
||||
else if (store->isValidStatePath(j->second.path))
|
||||
computeFSClosure(j->second.path, livePaths, true, true, 0);
|
||||
=======
|
||||
*/
|
||||
|
||||
string gcLevelStr = drv.env["__gcLevel"];
|
||||
int gcLevel;
|
||||
if (!string2Int(gcLevelStr,gcLevel)) {
|
||||
gcLevel = defaultGcLevel;
|
||||
}
|
||||
|
||||
if (gcLevel >= gcKeepOutputsThreshold)
|
||||
for (DerivationOutputs::iterator j = drv.outputs.begin();
|
||||
j != drv.outputs.end(); ++j)
|
||||
if (store->isValidPath(j->second.path) || store->isValidStatePath(j->second.path))
|
||||
computeFSClosure(j->second.path, livePaths, true, true, 0);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ bool readOnlyMode = false;
|
|||
string thisSystem = "unset";
|
||||
unsigned int maxSilentTime = 0;
|
||||
Paths substituters;
|
||||
bool useBuildHook = true;
|
||||
|
||||
|
||||
static bool settingsRead = false;
|
||||
|
|
|
|||
|
|
@ -80,6 +80,10 @@ extern unsigned int maxSilentTime;
|
|||
from a CD. */
|
||||
extern Paths substituters;
|
||||
|
||||
/* Whether to use build hooks (for distributed builds). Sometimes
|
||||
users want to disable this from the command-line. */
|
||||
extern bool useBuildHook;
|
||||
|
||||
Strings querySetting(const string & name, const Strings & def);
|
||||
|
||||
string querySetting(const string & name, const string & def);
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ RemoteStore::RemoteStore()
|
|||
unsigned int magic = readInt(from);
|
||||
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
|
||||
|
||||
unsigned int daemonVersion = readInt(from);
|
||||
daemonVersion = readInt(from);
|
||||
if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
||||
throw Error("Nix daemon protocol version not supported");
|
||||
writeInt(PROTOCOL_VERSION, to);
|
||||
|
|
@ -203,6 +203,8 @@ void RemoteStore::setOptions()
|
|||
writeInt(verbosity, to);
|
||||
writeInt(maxBuildJobs, to);
|
||||
writeInt(maxSilentTime, to);
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
|
||||
writeInt(useBuildHook, to);
|
||||
processStderr();
|
||||
}
|
||||
|
||||
|
|
@ -312,7 +314,9 @@ Path RemoteStore::queryDeriver(const Path & path)
|
|||
writeInt(wopQueryDeriver, to);
|
||||
writeString(path, to);
|
||||
processStderr();
|
||||
return readStorePath(from);
|
||||
Path drvPath = readString(from);
|
||||
if (drvPath != "") assertStorePath(drvPath);
|
||||
return drvPath;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -117,6 +117,7 @@ private:
|
|||
FdSink to;
|
||||
FdSource from;
|
||||
Pid child;
|
||||
unsigned int daemonVersion;
|
||||
|
||||
void processStderr(Sink * sink = 0, Source * source = 0);
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,9 @@ namespace nix {
|
|||
#define WORKER_MAGIC_1 0x6e697863
|
||||
#define WORKER_MAGIC_2 0x6478696f
|
||||
|
||||
#define PROTOCOL_VERSION 0x101
|
||||
#define PROTOCOL_VERSION 0x102
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
|
||||
|
||||
typedef enum {
|
||||
|
|
|
|||
|
|
@ -588,12 +588,17 @@ static void opExport(Strings opFlags, Strings opArgs)
|
|||
|
||||
static void opImport(Strings opFlags, Strings opArgs)
|
||||
{
|
||||
if (!opFlags.empty()) throw UsageError("unknown flag");
|
||||
bool requireSignature = false;
|
||||
for (Strings::iterator i = opFlags.begin();
|
||||
i != opFlags.end(); ++i)
|
||||
if (*i == "--require-signature") requireSignature = true;
|
||||
else throw UsageError(format("unknown flag `%1%'") % *i);
|
||||
|
||||
if (!opArgs.empty()) throw UsageError("no arguments expected");
|
||||
|
||||
FdSource source(STDIN_FILENO);
|
||||
while (readInt(source) == 1)
|
||||
cout << format("%1%\n") % store->importPath(false, source) << std::flush;
|
||||
cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -225,7 +225,8 @@ struct TunnelSource : Source
|
|||
};
|
||||
|
||||
|
||||
static void performOp(Source & from, Sink & to, unsigned int op)
|
||||
static void performOp(unsigned int clientVersion,
|
||||
Source & from, Sink & to, unsigned int op)
|
||||
{
|
||||
switch (op) {
|
||||
|
||||
|
|
@ -666,6 +667,8 @@ static void performOp(Source & from, Sink & to, unsigned int op)
|
|||
verbosity = (Verbosity) readInt(from);
|
||||
maxBuildJobs = readInt(from);
|
||||
maxSilentTime = readInt(from);
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
|
||||
useBuildHook = readInt(from) != 0;
|
||||
startWork();
|
||||
stopWork();
|
||||
break;
|
||||
|
|
@ -755,7 +758,7 @@ static void processConnection()
|
|||
|
||||
try {
|
||||
printMsg(lvlInfo, format("Processing op '%1%' with pid '%2%'") % op % myPid);
|
||||
performOp(from, to, op);
|
||||
performOp(clientVersion, from, to, op);
|
||||
printMsg(lvlInfo, format("Processed op '%1%'") % op);
|
||||
} catch (Error & e) {
|
||||
stopWork(false, e.msg());
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue