From dacf2e0e87cfd17cba342c159806e9ab0e27808a Mon Sep 17 00:00:00 2001 From: Wouter den Breejen Date: Mon, 8 Oct 2007 11:58:34 +0000 Subject: [PATCH] Merged R9105 --- Makefile.am | 4 +- doc/manual/nix-store.xml | 30 ---- externals/Makefile.am | 6 +- scripts/download-using-manifests.pl.in | 58 +++++-- scripts/nix-channel.in | 13 +- scripts/nix-pack-closure.in | 6 +- scripts/nix-prefetch-url.in | 67 ++++++-- scripts/nix-pull.in | 79 ++++----- src/libmain/shared.cc | 8 +- src/libstore/build.cc | 204 ++++++++++++++++------ src/libstore/db.cc | 13 +- src/libstore/db.hh | 3 +- src/libstore/gc.cc | 2 + src/libstore/globals.cc | 7 +- src/libstore/globals.hh | 5 + src/libstore/local-store.cc | 226 +++++-------------------- src/libstore/local-store.hh | 27 +-- src/libstore/misc.cc | 7 +- src/libstore/remote-store.cc | 14 +- src/libstore/remote-store.hh | 8 +- src/libstore/store-api.cc | 42 ++++- src/libstore/store-api.hh | 67 ++++---- src/libstore/worker-protocol.hh | 1 - src/libutil/types.hh | 26 +-- src/libutil/util.cc | 7 +- src/libutil/util.hh | 2 +- src/nix-store/help.txt | 2 - src/nix-store/nix-store.cc | 71 +------- 28 files changed, 503 insertions(+), 502 deletions(-) diff --git a/Makefile.am b/Makefile.am index d423ada61..41ad1f9bc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -35,11 +35,11 @@ init-state: $(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots $(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp $(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels - rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles - ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles + ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles $(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool $(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(prefix)/store $(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests + ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests # $(bindir)/nix-store --init else diff --git a/doc/manual/nix-store.xml b/doc/manual/nix-store.xml index 038fd523f..d787c337b 100644 --- a/doc/manual/nix-store.xml +++ b/doc/manual/nix-store.xml @@ -649,36 +649,6 @@ $ gv graph.ps - - - - - - Operation <option>--verify</option> diff --git a/externals/Makefile.am b/externals/Makefile.am index 06e8eedee..854a65268 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -35,12 +35,12 @@ endif # CWI ATerm -ATERM = aterm-2.4.2-fixes +ATERM = aterm-2.4.2-fixes-r2 $(ATERM).tar.bz2: @echo "Nix requires the CWI ATerm library to build." - @echo "Please download version 2.4.2-fixes from" - @echo " http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes.tar.bz2" + @echo "Please download version 2.4.2-fixes-r2 from" + @echo " http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes-r2.tar.bz2" @echo "and place it in the externals/ directory." false diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index 65ab36084..f963eead1 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -22,16 +22,6 @@ my $tmpNar2 = "$tmpDir/nar2"; END { unlink $tmpNar; unlink $tmpNar2; rmdir $tmpDir; } -# Check the arguments. -die unless scalar @ARGV == 1; -my $targetPath = $ARGV[0]; - -my $date = strftime ("%F %H:%M:%S UTC", gmtime (time)); -print LOGFILE "$$ get $targetPath $date\n"; - -print "\n*** Trying to download/patch `$targetPath'\n"; - - # Load all manifests. my %narFiles; my %localPaths; @@ -46,6 +36,54 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") { } +# Parse the arguments. + +if ($ARGV[0] eq "--query-paths") { + foreach my $storePath (keys %narFiles) { print "$storePath\n"; } + foreach my $storePath (keys %localPaths) { print "$storePath\n"; } + exit 0; +} + +elsif ($ARGV[0] eq "--query-info") { + shift @ARGV; + foreach my $storePath (@ARGV) { + my $info; + if (defined $narFiles{$storePath}) { + $info = @{$narFiles{$storePath}}[0]; + } + elsif (defined $localPaths{$storePath}) { + $info = @{$localPaths{$storePath}}[0]; + } + else { + next; # not an error + } + print "$storePath\n"; + print "$info->{deriver}\n"; + my @references = split " ", $info->{references}; + my $count = scalar @references; + print "$count\n"; + foreach my $reference (@references) { + print "$reference\n"; + } + } + exit 0; +} + +elsif ($ARGV[0] ne "--substitute") { + die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n"; +} + + +die unless scalar @ARGV == 2; +my $targetPath = $ARGV[1]; + + +my $date = strftime ("%F %H:%M:%S UTC", gmtime (time)); +print LOGFILE "$$ get $targetPath $date\n"; + +print "\n*** Trying to download/patch `$targetPath'\n"; + + # If we can copy from a local path, do that. my $localPathList = $localPaths{$targetPath}; foreach my $localPath (@{$localPathList}) { diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in index 41a75adf1..1ef9c5844 100644 --- a/scripts/nix-channel.in +++ b/scripts/nix-channel.in @@ -8,6 +8,12 @@ my $stateDir = $ENV{"NIX_STATE_DIR"}; $stateDir = "@localstatedir@/nix" unless defined $stateDir; +# Turn on caching in nix-prefetch-url. +my $channelCache = "$stateDir/channel-cache"; +$ENV{'NIX_DOWNLOAD_CACHE'} = $channelCache; +mkdir $channelCache, 0755 unless -e $channelCache; + + # Figure out the name of the `.nix-channels' file to use. my $home = $ENV{"HOME"}; die '$HOME not set' unless defined $home; @@ -70,10 +76,6 @@ sub removeChannel { sub update { readChannels; - # Get rid of all the old substitutes. - system("@bindir@/nix-store", "--clear-substitutes") == 0 - or die "cannot clear substitutes"; - # Remove all the old manifests. for my $manifest (glob "$stateDir/manifests/*.nixmanifest") { unlink $manifest or die "cannot remove `$manifest': $!"; @@ -98,7 +100,8 @@ sub update { my $fullURL = "$url/nixexprs.tar.bz2"; print "downloading Nix expressions from `$fullURL'...\n"; $ENV{"PRINT_PATH"} = 1; - my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`; + $ENV{"QUIET"} = 1; + my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL'`; die "cannot fetch `$fullURL'" if $? != 0; chomp $path; $inputs .= '"' . $channelName . '"' . " " . $path . " "; diff --git a/scripts/nix-pack-closure.in b/scripts/nix-pack-closure.in index 8d9ecf126..0634537d0 100644 --- a/scripts/nix-pack-closure.in +++ b/scripts/nix-pack-closure.in @@ -17,9 +17,9 @@ $binDir = "@bindir@" unless defined $binDir; my $tmpDir = tempdir("nix-pack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; -mkdir "$tmpDir/contents", 0777 or die; -mkdir "$tmpDir/references", 0777 or die; -mkdir "$tmpDir/derivers", 0777 or die; +mkdir "$tmpDir/contents", 0755 or die; +mkdir "$tmpDir/references", 0755 or die; +mkdir "$tmpDir/derivers", 0755 or die; open TOPLEVEL, ">$tmpDir/top-level" or die; diff --git a/scripts/nix-prefetch-url.in b/scripts/nix-prefetch-url.in index 2b4478501..186804d8e 100644 --- a/scripts/nix-prefetch-url.in +++ b/scripts/nix-prefetch-url.in @@ -36,6 +36,12 @@ if test -n "$expHash"; then fi +doDownload() { + @curl@ $cacheFlags --fail -# --show-error --location --max-redirs 20 --disable-epsv \ + --cookie-jar $tmpPath/cookies "$url" -o $tmpFile +} + + # If we don't know the hash or a file with that hash doesn't exist, # download the file and add it to the store. if test -z "$finalPath"; then @@ -44,22 +50,61 @@ if test -z "$finalPath"; then tmpFile=$tmpPath/$name mkdir $tmpPath + # Optionally do timestamp-based caching of the download. + # Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is + # the hash and the timestamp of the file at $url. The caching of + # the file *contents* is done in Nix store, where it can be + # garbage-collected independently. + if test -n "$NIX_DOWNLOAD_CACHE"; then + echo -n "$url" > $tmpPath/url + urlHash=$(nix-hash --type sha256 --base32 --flat $tmpPath/url) + echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url" + cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType" + cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp" + cacheFlags="--remote-time" + if test -e "$cachedTimestampFN" -a -e "$cachedHashFN"; then + # Only download the file if it is newer than the cached version. + cacheFlags="$cacheFlags --time-cond $cachedTimestampFN" + fi + fi + # Perform the download. - @curl@ --fail --location --max-redirs 20 --disable-epsv \ - --cookie-jar $tmpPath/cookies "$url" > $tmpFile + doDownload - # Compute the hash. - hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile) - if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi + if test -n "$NIX_DOWNLOAD_CACHE" -a ! -e $tmpFile; then + # Curl didn't create $tmpFile, so apparently there's no newer + # file on the server. + hash=$(cat $cachedHashFN) + finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$hash" "$name") + if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then + echo "cached contents of \`$url' disappeared, redownloading..." >&2 + finalPath= + cacheFlags="--remote-time" + doDownload + fi + fi - # Add the downloaded file to the Nix store. - finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile) + if test -z "$finalPath"; then - if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi + # Compute the hash. + hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile) + if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi - if test -n "$expHash" -a "$expHash" != "$hash"; then - echo "hash mismatch for URL \`$url'" >&2 - exit 1 + if test -n "$NIX_DOWNLOAD_CACHE"; then + echo $hash > $cachedHashFN + touch -r $tmpFile $cachedTimestampFN + fi + + # Add the downloaded file to the Nix store. + finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile) + + if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi + + if test -n "$expHash" -a "$expHash" != "$hash"; then + echo "hash mismatch for URL \`$url'" >&2 + exit 1 + fi + fi fi diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index db0594040..f1f01f4d2 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -7,8 +7,6 @@ use readmanifest; my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; -my $manifest = "$tmpDir/manifest"; - my $binDir = $ENV{"NIX_BIN_DIR"}; $binDir = "@bindir@" unless defined $binDir; @@ -38,16 +36,43 @@ my %patches; my $skipWrongStore = 0; +sub downloadFile { + my $url = shift; + $ENV{"PRINT_PATH"} = 1; + $ENV{"QUIET"} = 1; + my ($dummy, $path) = `@bindir@/nix-prefetch-url '$url'`; + chomp $path; + return $path; +} + sub processURL { my $url = shift; $url =~ s/\/$//; - print "obtaining list of Nix archives at $url...\n"; - system("@curl@ --fail --silent --show-error --location --max-redirs 20 " . - "'$url' > '$manifest'") == 0 - or die "curl failed: $?"; + my $manifest; + # First see if a bzipped manifest is available. + if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) { + print "obtaining list of Nix archives at `$url.bz2'...\n"; + my $bzipped = downloadFile "$url.bz2"; + + $manifest = "$tmpDir/MANIFEST"; + + system("@bunzip2@ < $bzipped > $manifest") == 0 + or die "cannot decompress manifest"; + + $manifest = (`$binDir/nix-store --add $manifest` + or die "cannot copy $manifest to the store"); + chomp $manifest; + } + + # Otherwise, just get the uncompressed manifest. + else { + print "obtaining list of Nix archives at `$url'...\n"; + $manifest = downloadFile $url; + } + if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) { die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n"; } @@ -72,8 +97,8 @@ sub processURL { my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest"; - system ("@coreutils@/mv", "-f", "$manifest", "$finalPath") == 0 - or die "cannot move `$manifest' to `$finalPath"; + system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0 + or die "cannot link `$finalPath to `$manifest'"; } while (@ARGV) { @@ -88,41 +113,3 @@ while (@ARGV) { my $size = scalar (keys %narFiles) + scalar (keys %localPaths); print "$size store paths in manifest\n"; - - -# Register all substitutes. -print STDERR "registering substitutes...\n"; - -my $pid = open(WRITE, "|$binDir/nix-store --register-substitutes") - or die "cannot run nix-store"; - -sub writeRegistration { - my $storePath = shift; - my $object = shift; - print WRITE "$storePath\n"; - print WRITE "$object->{deriver}\n"; - print WRITE "$libexecDir/nix/download-using-manifests.pl\n"; - print WRITE "0\n"; - my @references = split " ", $object->{references}; - my $count = scalar @references; - print WRITE "$count\n"; - foreach my $reference (@references) { - print WRITE "$reference\n"; - } -} - -foreach my $storePath (keys %narFiles) { - my $narFileList = $narFiles{$storePath}; - foreach my $narFile (@{$narFileList}) { - writeRegistration $storePath, $narFile; - } -} - -foreach my $storePath (keys %localPaths) { - my $localPathList = $localPaths{$storePath}; - foreach my $localPath (@{$localPathList}) { - writeRegistration $storePath, $localPath; - } -} - -close WRITE or die "nix-store failed: $?"; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index b6c388afd..472e38ae7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -116,6 +116,12 @@ static void initAndRun(int argc, char * * argv) nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); + string subs = getEnv("NIX_SUBSTITUTERS", "default"); + if (subs == "default") + substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl"); + else + substituters = tokenizeString(subs, ":"); + /* Get some settings from the configuration file. */ thisSystem = querySetting("system", SYSTEM); maxBuildJobs = queryIntSetting("build-max-jobs", 1); @@ -331,7 +337,7 @@ int main(int argc, char * * argv) "Try `%2% --help' for more information.") % e.what() % programId); return 1; - } catch (Error & e) { + } catch (BaseError & e) { printMsg(lvlError, format("error: %1%") % e.msg()); return 1; } catch (std::exception & e) { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ace5e473d..b1c01abb2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -166,6 +166,11 @@ private: /* Goals waiting for a build slot. */ WeakGoals wantingToBuild; + /* Goals waiting for info from substituters (using --query-info), + and the info they're (collectively) waiting for. */ + WeakGoals waitingForInfo; + map requestedInfo; + /* Child processes currently running. */ Children children; @@ -214,12 +219,24 @@ public: /* Put `goal' to sleep until a child process terminates, i.e., a call is made to childTerminate(..., true). */ void waitForChildTermination(GoalPtr goal); + + /* Put `goal' to sleep until the top-level loop has run `sub' to + get info about `storePath' (with --query-info). We combine + substituter invocations to reduce overhead. */ + void waitForInfo(GoalPtr goal, Path sub, Path storePath); /* Loop until the specified top-level goals have finished. */ void run(const Goals & topGoals); /* Wait for input to become available. */ void waitForInput(); + +private: + + /* Process the pending paths in requestedInfo and wake up the + goals in waitingForInfo. */ + void getInfo(); + }; @@ -797,7 +814,7 @@ void DerivationGoal::haveDerivation() substitutes. */ if (store->hasSubstitutes(*i)) addWaitee(worker.makeSubstitutionGoal(*i)); - + if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); else @@ -1952,21 +1969,23 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid) class SubstitutionGoal : public Goal { + friend class Worker; + private: /* The store path that should be realised through a substitute. */ Path storePath; //TODO !!!!!!!!!!!!!!!!!!!!! add statePath? - /* The remaining substitutes for this path. */ - Substitutes subs; + /* The remaining substituters. */ + Paths subs; - /* The current substitute. */ - Substitute sub; + /* The current substituter. */ + Path sub; - /* Outgoing references for this path. */ + /* Path info returned by the substituter's --query-info operation. */ + bool infoOkay; PathSet references; - - /* Outgoing state references for this path. */ - PathSet stateReferences; + PathSet stateReferences; /* Outgoing state references for this path. */ + Path deriver; /* Pipe for the substitute's standard output/error. */ Pipe logPipe; @@ -1990,8 +2009,9 @@ public: /* The states. */ void init(); - void referencesValid(); void tryNext(); + void gotInfo(); + void referencesValid(); void tryToRun(); void finished(); @@ -2049,17 +2069,46 @@ void SubstitutionGoal::init() return; } - /* !!! race condition; should get the substitutes and the - references in a transaction (in case a clearSubstitutes() is - done simultaneously). */ + subs = substituters; - /* Read the substitutes. */ - subs = store->querySubstitutes(storePath); + tryNext(); +} + +void SubstitutionGoal::tryNext() +{ + trace("trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + printMsg(lvlError, + format("path `%1%' is required, but there is no substituter that can build it") + % storePath); + amDone(ecFailed); + return; + } + + sub = subs.front(); + subs.pop_front(); + + infoOkay = false; + state = &SubstitutionGoal::gotInfo; + worker.waitForInfo(shared_from_this(), sub, storePath); +} + + +void SubstitutionGoal::gotInfo() +{ + trace("got info"); + + if (!infoOkay) { + tryNext(); + return; + } + /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - store->queryStoreReferences(storePath, references, 0); - for (PathSet::iterator i = references.begin(); i != references.end(); ++i) if (*i != storePath) /* ignore self-references */ @@ -2074,7 +2123,7 @@ void SubstitutionGoal::init() void SubstitutionGoal::referencesValid() { - trace("all referenced realised"); + trace("all references realised"); if (nrFailed > 0) { printMsg(lvlError, @@ -2087,28 +2136,7 @@ void SubstitutionGoal::referencesValid() i != references.end(); ++i) if (*i != storePath) /* ignore self-references */ assert(store->isValidPath(*i)); - - tryNext(); -} - -void SubstitutionGoal::tryNext() -{ - trace("trying next substitute"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - printMsg(lvlError, - format("path `%1%' is required, but it has no (remaining) substitutes") - % storePath); - amDone(ecFailed); - return; - } - sub = subs.front(); - subs.pop_front(); - - /* Wait until we can run the substitute program. */ state = &SubstitutionGoal::tryToRun; worker.waitForBuildSlot(shared_from_this()); } @@ -2139,7 +2167,7 @@ void SubstitutionGoal::tryToRun() printMsg(lvlInfo, format("substituting path `%1%' using substituter `%2%'") - % storePath % sub.program); + % storePath % sub); logPipe.create(); @@ -2170,14 +2198,15 @@ void SubstitutionGoal::tryToRun() commonChildInit(logPipe); /* Fill in the arguments. */ - Strings args(sub.args); - args.push_front(storePath); - args.push_front(baseNameOf(sub.program)); + Strings args; + args.push_back(baseNameOf(sub)); + args.push_back("--substitute"); + args.push_back(storePath); const char * * argArr = strings2CharPtrs(args); - execv(sub.program.c_str(), (char * *) argArr); + execv(sub.c_str(), (char * *) argArr); - throw SysError(format("executing `%1%'") % sub.program); + throw SysError(format("executing `%1%'") % sub); } catch (std::exception & e) { std::cerr << format("substitute error: %1%\n") % e.what(); @@ -2230,7 +2259,7 @@ void SubstitutionGoal::finished() printMsg(lvlInfo, format("substitution of path `%1%' using substituter `%2%' failed: %3%") - % storePath % sub.program % e.msg()); + % storePath % sub % e.msg()); /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; @@ -2245,7 +2274,7 @@ void SubstitutionGoal::finished() Transaction txn; createStoreTransaction(txn); registerValidPath(txn, storePath, contentHash, //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! how about substituing a state path ????? - references, stateReferences, sub.deriver, 0); + references, stateReferences, deriver, 0); txn.commit(); outputLock->setDeletion(true); @@ -2430,6 +2459,76 @@ void Worker::waitForChildTermination(GoalPtr goal) } +void Worker::waitForInfo(GoalPtr goal, Path sub, Path storePath) +{ + debug("wait for info"); + requestedInfo[sub].insert(storePath); + waitingForInfo.insert(goal); +} + + +void Worker::getInfo() +{ + for (map::iterator i = requestedInfo.begin(); + i != requestedInfo.end(); ++i) + { + Path sub = i->first; + PathSet paths = i->second; + + while (!paths.empty()) { + + /* Run the substituter for at most 100 paths at a time to + prevent command line overflows. */ + PathSet paths2; + while (!paths.empty() && paths2.size() < 100) { + paths2.insert(*paths.begin()); + paths.erase(paths.begin()); + } + + /* Ask the substituter for the references and deriver of + the paths. */ + debug(format("running `%1%' to get info about `%2%'") % sub % showPaths(paths2)); + Strings args; + args.push_back("--query-info"); + args.insert(args.end(), paths2.begin(), paths2.end()); + string res = runProgram(sub, false, args); + std::istringstream str(res); + + while (true) { + ValidPathInfo info = decodeValidPathInfo(str); + if (info.path == "") break; + + /* !!! inefficient */ + for (WeakGoals::iterator k = waitingForInfo.begin(); + k != waitingForInfo.end(); ++k) + { + GoalPtr goal = k->lock(); + if (goal) { + SubstitutionGoal * goal2 = dynamic_cast(goal.get()); + if (goal2->storePath == info.path) { + goal2->references = info.references; + goal2->deriver = info.deriver; + goal2->infoOkay = true; + wakeUp(goal); + } + } + } + } + } + } + + for (WeakGoals::iterator k = waitingForInfo.begin(); + k != waitingForInfo.end(); ++k) + { + GoalPtr goal = k->lock(); + if (goal) wakeUp(goal); + } + + requestedInfo.clear(); + waitingForInfo.clear(); // !!! have we done them all? +} + + void Worker::run(const Goals & _topGoals) { for (Goals::iterator i = _topGoals.begin(); @@ -2456,11 +2555,14 @@ void Worker::run(const Goals & _topGoals) if (topGoals.empty()) break; - /* !!! not when we're polling */ - assert(!children.empty()); - + getInfo(); + /* Wait for input. */ - waitForInput(); + if (!children.empty()) + waitForInput(); + else + /* !!! not when we're polling */ + assert(!awake.empty()); } /* If --keep-going is not set, it's possible that the main goal diff --git a/src/libstore/db.cc b/src/libstore/db.cc index 8370d1cd3..abeeeee0f 100644 --- a/src/libstore/db.cc +++ b/src/libstore/db.cc @@ -197,7 +197,8 @@ void Database::open2(const string & path, bool removeOldEnv) env->set_errcall(errorPrinter); env->set_msgcall(messagePrinter); - //env->set_verbose(DB_VERB_REGISTER, 1); + if (getEnv("NIX_DEBUG_DB_REGISTER") == "1") + env->set_verbose(DB_VERB_REGISTER, 1); env->set_verbose(DB_VERB_RECOVERY, 1); /* Smaller log files. */ @@ -458,4 +459,14 @@ void Database::enumTable(const Transaction & txn, TableId table, +void Database::clearTable(const Transaction & txn, TableId table) +{ + try { + Db * db = getDb(table); + u_int32_t count; + db->truncate(txn.txn, &count, 0); + } catch (DbException e) { rethrow(e); } +} + + } diff --git a/src/libstore/db.hh b/src/libstore/db.hh index 496a07f90..d8330a608 100644 --- a/src/libstore/db.hh +++ b/src/libstore/db.hh @@ -91,7 +91,8 @@ public: void enumTable(const Transaction & txn, TableId table, Strings & keys, const string & keyPrefix = ""); - + + void clearTable(const Transaction & txn, TableId table); }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 758f86a4b..59d039349 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -607,6 +607,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, } #endif + if (!pathExists(*i)) continue; + printMsg(lvlInfo, format("deleting `%1%'") % *i); /* Okay, it's safe to delete. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d0285676f..e473f6799 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -28,9 +28,12 @@ unsigned int maxBuildJobs = 1; bool readOnlyMode = false; string thisSystem = "unset"; unsigned int maxSilentTime = 0; -static bool settingsRead = false; -uid_t callingUID = 0; //A root user will not set this value, so the default uid is 0 +Paths substituters; + +static bool settingsRead = false; + +uid_t callingUID = 0; //A root user will not set this value, so the default uid is 0 bool singleThreaded = false; //TODO Gives an error: cannot start worker (environment already open) / waiting for process 7487: No child processes bool sendOutput = true; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0846f4e07..b163f127e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -75,6 +75,11 @@ extern string thisSystem; infinity. */ extern unsigned int maxSilentTime; +/* The substituters. There are programs that can somehow realise a + store path without building, e.g., by downloading it or copying it + from a CD. */ +extern Paths substituters; + Strings querySetting(const string & name, const Strings & def); string querySetting(const string & name, const string & def); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0028c88d3..756a59f71 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -79,22 +79,6 @@ static TableId dbStateStateReferences = 0; */ static TableId dbSolidStateReferences = 0; -/* dbSubstitutes :: Path -> [[Path]] - - Each pair $(p, subs)$ tells Nix that it can use any of the - substitutes in $subs$ to build path $p$. Each substitute defines a - command-line invocation of a program (i.e., the first list element - is the full path to the program, the remaining elements are - arguments). - - The main purpose of this is for distributed caching of derivates. - One system can compute a derivate and put it on a website (as a Nix - archive), for instance, and then another system can register a - substitute for that derivate. The substitute in this case might be - a Nix derivation that fetches the Nix archive. -*/ -static TableId dbSubstitutes = 0; - /* dbDerivers :: Path -> [Path] This table lists the derivation used to build a path. There can @@ -174,12 +158,6 @@ static TableId dbStateSnapshots = 0; */ static TableId dbSharedState = 0; -bool Substitute::operator == (const Substitute & sub) const -{ - return program == sub.program - && args == sub.args; -} - static void upgradeStore07(); static void upgradeStore09(); @@ -205,6 +183,8 @@ void checkStoreNotSymlink() LocalStore::LocalStore(bool reserveSpace) { + substitutablePathsLoaded = false; + if (readOnlyMode) return; checkStoreNotSymlink(); @@ -234,7 +214,6 @@ LocalStore::LocalStore(bool reserveSpace) } dbValidPaths = nixDB.openTable("validpaths"); dbValidStatePaths = nixDB.openTable("validpaths_state"); - dbSubstitutes = nixDB.openTable("substitutes"); dbDerivers = nixDB.openTable("derivers"); dbStateInfo = nixDB.openTable("stateinfo"); @@ -397,6 +376,7 @@ bool isValidStatePathTxn(const Transaction & txn, const Path & path) return nixDB.queryString(txn, dbValidStatePaths, path, s); } + bool LocalStore::isValidStatePath(const Path & path) { return isValidStatePathTxn(noTxn, path); @@ -412,11 +392,8 @@ bool LocalStore::isValidComponentOrStatePath(const Path & path) return isValidComponentOrStatePathTxn(noTxn, path); } - -static Substitutes readSubstitutes(const Transaction & txn, - const Path & srcPath); - - +//TODO REMOVE BLOCK +/* static bool isRealisablePath(const Transaction & txn, const Path & path) { return isValidPathTxn(txn, path) || readSubstitutes(txn, path).size() > 0; @@ -427,10 +404,12 @@ static bool isRealisableStatePath(const Transaction & txn, const Path & path) return isValidStatePathTxn(txn, path) || readSubstitutes(txn, path).size() > 0; } + static bool isRealisableComponentOrStatePath(const Transaction & txn, const Path & path) { return isValidComponentOrStatePathTxn(txn, path) || readSubstitutes(txn, path).size() > 0; //TODO State paths are not yet in substitutes !!!!!!!!!!!!!! ?? } +*/ /* static string addPrefix(const string & prefix, const string & s) @@ -456,8 +435,9 @@ static string stripPrefix(const string & prefix, const string & s) void setReferences(const Transaction & txn, const Path & store_or_statePath, const PathSet & references, const PathSet & stateReferences, const unsigned int revision) { + /* For unrealisable paths, we can only clear the references. */ - if (references.size() > 0 && !isRealisableComponentOrStatePath(txn, store_or_statePath)) + if (references.size() > 0 && !isValidComponentOrStatePathTxn(txn, store_or_statePath)) throw Error(format("cannot set references for path `%1%' which is invalid and has no substitutes") % store_or_statePath); @@ -468,7 +448,7 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath, printMsg(lvlError, format("'%2%' has stateReferences: %1%") % *i % store_or_statePath); */ - if(isRealisablePath(txn, store_or_statePath)) + if(isValidPathTxn(txn, store_or_statePath)) { printMsg(lvlError, format("Setting references for storepath '%1%'") % store_or_statePath); @@ -486,7 +466,7 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath, nixDB.setStrings(txn, dbComponentComponentReferences, store_or_statePath, Paths(references.begin(), references.end())); nixDB.setStrings(txn, dbComponentStateReferences, store_or_statePath, Paths(stateReferences.begin(), stateReferences.end())); } - else if(isRealisableStatePath(txn, store_or_statePath)) + else if(isValidStatePathTxn(txn, store_or_statePath)) { printMsg(lvlError, format("Setting references for statepath '%1%' (revision:%2%)") % store_or_statePath % unsignedInt2String(revision)); @@ -525,9 +505,9 @@ void queryXReferencesTxn(const Transaction & txn, const Path & store_or_statePat table2 = dbStateStateReferences; } - if(isRealisablePath(txn, store_or_statePath)) + if(isValidPathTxn(txn, store_or_statePath)) nixDB.queryStrings(txn, table1, store_or_statePath, references2); - else if(isRealisableStatePath(txn, store_or_statePath)){ + else if(isValidStatePathTxn(txn, store_or_statePath)){ Path statePath_ns = toNonSharedPathTxn(txn, store_or_statePath); //Lookup its where it points to if its shared queryStateReferences(nixDB, txn, table2, dbStateRevisions, statePath_ns, references2, revision, timestamp); } @@ -655,7 +635,7 @@ static PathSet getStateReferrersTxn(const Transaction & txn, const Path & store_ void queryStoreReferrersTxn(const Transaction & txn, const Path & storePath, PathSet & referrers, const unsigned int revision) { - if (!isRealisableComponentOrStatePath(txn, storePath)) + if (!isValidComponentOrStatePathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); PathSet referrers2 = getStoreReferrersTxn(txn, storePath, revision); referrers.insert(referrers2.begin(), referrers2.end()); @@ -669,7 +649,7 @@ void LocalStore::queryStoreReferrers(const Path & storePath, void queryStateReferrersTxn(const Transaction & txn, const Path & storePath, PathSet & stateReferrers, const unsigned int revision) { - if (!isRealisableComponentOrStatePath(txn, storePath)) + if (!isValidComponentOrStatePathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); PathSet stateReferrers2 = getStateReferrersTxn(txn, storePath, revision); stateReferrers.insert(stateReferrers2.begin(), stateReferrers2.end()); @@ -686,7 +666,7 @@ void setDeriver(const Transaction & txn, const Path & storePath, const Path & de if (deriver == "") return; assertStorePath(deriver); - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); if (isStateDrvPathTxn(txn, deriver)){ //Redirect if its a state component @@ -740,7 +720,7 @@ void addStateDeriver(const Transaction & txn, const Path & storePath, const Path return; assertStorePath(deriver); - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); Derivation drv = derivationFromPathTxn(txn, deriver); @@ -793,7 +773,7 @@ bool isStateDrv(const Derivation & drv) static Path queryDeriver(const Transaction & txn, const Path & storePath) { - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); Path deriver; @@ -817,7 +797,7 @@ Path LocalStore::queryDeriver(const Path & path) //A '*' as argument stands for all identifiers or all users PathSet queryDerivers(const Transaction & txn, const Path & storePath, const string & identifier, const string & user) { - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); if(user == "") @@ -845,7 +825,7 @@ PathSet queryDerivers(const Transaction & txn, const Path & storePath, const str return filtereddata; } - + PathSet LocalStore::queryDerivers(const Path & storePath, const string & identifier, const string & user) { return nix::queryDerivers(noTxn, storePath, identifier, user); @@ -865,120 +845,33 @@ PathSet queryDeriversStatePath(const Transaction & txn, const Path & storePath, } */ - -const int substituteVersion = 2; - - -static Substitutes readSubstitutes(const Transaction & txn, - const Path & srcPath) +PathSet LocalStore::querySubstitutablePaths() { - Strings ss; - nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss); - - Substitutes subs; - - for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { - if (i->size() < 4 || (*i)[3] != 0) { - /* Old-style substitute. !!! remove this code - eventually? */ - break; + if (!substitutablePathsLoaded) { + for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) { + debug(format("running `%1%' to find out substitutable paths") % *i); + Strings args; + args.push_back("--query-paths"); + Strings ss = tokenizeString(runProgram(*i, false, args), "\n"); + for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) { + if (!isStorePath(*j)) + throw Error(format("`%1%' returned a bad substitutable path `%2%'") + % *i % *j); + substitutablePaths.insert(*j); + } } - Strings ss2 = unpackStrings(*i); - if (ss2.size() == 0) continue; - int version; - if (!string2Int(ss2.front(), version)) continue; - if (version != substituteVersion) continue; - if (ss2.size() != 4) throw Error("malformed substitute"); - Strings::iterator j = ss2.begin(); - j++; - Substitute sub; - sub.deriver = *j++; - sub.program = *j++; - sub.args = unpackStrings(*j++); - subs.push_back(sub); + substitutablePathsLoaded = true; } - return subs; + return substitutablePaths; } -static void writeSubstitutes(const Transaction & txn, - const Path & srcPath, const Substitutes & subs) +bool LocalStore::hasSubstitutes(const Path & path) { - Strings ss; - - for (Substitutes::const_iterator i = subs.begin(); - i != subs.end(); ++i) - { - Strings ss2; - ss2.push_back((format("%1%") % substituteVersion).str()); - ss2.push_back(i->deriver); - ss2.push_back(i->program); - ss2.push_back(packStrings(i->args)); - ss.push_back(packStrings(ss2)); - } - - nixDB.setStrings(txn, dbSubstitutes, srcPath, ss); -} - - -void registerSubstitute(const Transaction & txn, - const Path & srcPath, const Substitute & sub) -{ - assertStorePath(srcPath); - - Substitutes subs = readSubstitutes(txn, srcPath); - - if (find(subs.begin(), subs.end(), sub) != subs.end()) - return; - - /* New substitutes take precedence over old ones. If the - substitute is already present, it's moved to the front. */ - remove(subs.begin(), subs.end(), sub); - subs.push_front(sub); - - writeSubstitutes(txn, srcPath, subs); -} - - -Substitutes querySubstitutes(const Transaction & txn, const Path & path) -{ - return readSubstitutes(txn, path); -} - - -Substitutes LocalStore::querySubstitutes(const Path & path) -{ - return nix::querySubstitutes(noTxn, path); -} - - -static void invalidateStorePath(Transaction & txn, const Path & path); - - -void clearSubstitutes() //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ALSO FOR STATE -{ - Transaction txn(nixDB); - - /* Iterate over all paths for which there are substitutes. */ - Paths subKeys; - nixDB.enumTable(txn, dbSubstitutes, subKeys); - for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { - - /* Delete all substitutes for path *i. */ - nixDB.delPair(txn, dbSubstitutes, *i); - - /* Maintain the cleanup invariant. */ - if (!isValidPathTxn(txn, *i)) - invalidateStorePath(txn, *i); - } - - /* !!! there should be no referrers to any of the invalid - substitutable paths. This should be the case by construction - (the only referrers can be other invalid substitutable paths, - which have all been removed now). */ - - txn.commit(); + if (!substitutablePathsLoaded) + querySubstitutablePaths(); + return substitutablePaths.find(path) != substitutablePaths.end(); } @@ -1510,38 +1403,9 @@ void verifyStore(bool checkContents) } } - - printMsg(lvlInfo, "checking path realisability"); - - /* "Realisable" paths are those that are valid or have a - substitute. */ - PathSet realisablePaths(validPaths); - - - //TODO !!!!!!!!!!!!!!!!!!!!!!!!! Do also for validStatePaths - - - /* Check that the values of the substitute mappings are valid - paths. */ - Paths subKeys; - nixDB.enumTable(txn, dbSubstitutes, subKeys); - for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { - Substitutes subs = readSubstitutes(txn, *i); - if (!isStorePath(*i)) { - printMsg(lvlError, format("removing substitutes for non-store path `%1%'") % *i); - nixDB.delPair(txn, dbSubstitutes, *i); - } - else if (subs.size() == 0) - nixDB.delPair(txn, dbSubstitutes, *i); - else - realisablePaths.insert(*i); - } - - /* Check the cleanup invariant: only realisable paths can have `references', `referrers', or `derivers' entries. */ - /* Check the `derivers' table. */ printMsg(lvlInfo, "checking the derivers table"); Paths deriversKeys; @@ -1549,8 +1413,8 @@ void verifyStore(bool checkContents) for (Paths::iterator i = deriversKeys.begin(); i != deriversKeys.end(); ++i) { - if (realisablePaths.find(*i) == realisablePaths.end()) { - printMsg(lvlError, format("removing deriver entry for unrealisable path `%1%'") + if (validPaths.find(*i) == validPaths.end()) { + printMsg(lvlError, format("removing deriver entry for invalid path `%1%'") % *i); nixDB.delPair(txn, dbDerivers, *i); } @@ -1572,13 +1436,12 @@ void verifyStore(bool checkContents) for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i) { - if (realisablePaths.find(*i) == realisablePaths.end()) { - printMsg(lvlError, format("removing references entry for unrealisable path `%1%'") + if (validPaths.find(*i) == validPaths.end()) { + printMsg(lvlError, format("removing references entry for invalid path `%1%'") % *i); setReferences(txn, *i, PathSet(), PathSet(), 0); //TODO? } else { - bool isValid = validPaths.find(*i) != validPaths.end(); PathSet references; queryXReferencesTxn(txn, *i, references, true, -1); //TODO for (PathSet::iterator j = references.begin(); @@ -1594,7 +1457,7 @@ void verifyStore(bool checkContents) } */ - if (isValid && validPaths.find(*j) == validPaths.end()) { + if (validPaths.find(*j) == validPaths.end()) { printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") % *i % *j); } @@ -1629,6 +1492,7 @@ void setStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePath } } + void LocalStore::setStatePathsInterval(const PathSet & statePaths, const IntVector & intervals, bool allZero) { Transaction txn(nixDB); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9fdf307fb..97fdb5c20 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -23,6 +23,10 @@ extern string drvsLogDir; class LocalStore : public StoreAPI { +private: + bool substitutablePathsLoaded; + PathSet substitutablePaths; + public: /* Open the database environment. If `reserveSpace' is true, make @@ -45,8 +49,6 @@ public: bool isValidComponentOrStatePath(const Path & path); - Substitutes querySubstitutes(const Path & srcPath); - Hash queryPathHash(const Path & path); Path queryStatePathDrv(const Path & statePath); @@ -61,6 +63,10 @@ public: Path queryDeriver(const Path & path); + PathSet querySubstitutablePaths(); + + bool hasSubstitutes(const Path & path); + Path addToStore(const Path & srcPath, bool fixed = false, bool recursive = false, string hashAlgo = "", PathFilter & filter = defaultPathFilter); @@ -128,13 +134,6 @@ void createStoreTransaction(Transaction & txn); /* Copy a path recursively. */ void copyPath(const Path & src, const Path & dst); -/* Register a substitute. */ -void registerSubstitute(const Transaction & txn, - const Path & srcPath, const Substitute & sub); - -/* Deregister all substitutes. */ -void clearSubstitutes(); - /* Register the validity of a path, i.e., that `path' exists, that the paths referenced by it exists, and in the case of an output path of a derivation, that it has been produced by a succesful execution of @@ -146,16 +145,6 @@ void registerValidPath(const Transaction & txn, const PathSet & references, const PathSet & stateReferences, const Path & deriver, const unsigned int revision); -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash hash; - PathSet references; - PathSet stateReferences; - int unsigned revision; -}; - typedef list ValidPathInfos; void registerValidPaths(const Transaction & txn, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f699d6810..c81a31487 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -124,8 +124,7 @@ void queryMissing(const PathSet & targets, bool mustBuild = false; for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - if (!store->isValidPath(i->second.path) && - !store->hasSubstitutes(i->second.path)) + if (!store->isValidPath(i->second.path) && !store->hasSubstitutes(i->second.path)) mustBuild = true; if (mustBuild) { @@ -144,8 +143,8 @@ void queryMissing(const PathSet & targets, if (store->isValidPath(p)) continue; if (store->hasSubstitutes(p)) willSubstitute.insert(p); - PathSet refs; - store->queryStoreReferences(p, todo, 0); //TODO? + // XXX call the substituters + // store->queryReferences(p, todo); } } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e31e94135..518678515 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -182,12 +182,6 @@ bool RemoteStore::isValidComponentOrStatePath(const Path & path) return reply != 0; } -Substitutes RemoteStore::querySubstitutes(const Path & path) -{ - throw Error("not implemented 2"); -} - - bool RemoteStore::hasSubstitutes(const Path & path) { writeInt(wopHasSubstitutes, to); @@ -270,6 +264,12 @@ Path RemoteStore::queryDeriver(const Path & path) } +PathSet RemoteStore::querySubstitutablePaths() +{ + throw Error("not implemented"); +} + + Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, bool recursive, string hashAlgo, PathFilter & filter) { @@ -283,8 +283,6 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, dumpPath(srcPath, to, filter); processStderr(); return readStorePath(from); - //Path path = readStorePath(from); //TODO REMOVE CODE - //return path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index abd171b26..0ef3e730a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -31,10 +31,6 @@ public: bool isValidComponentOrStatePath(const Path & path); - Substitutes querySubstitutes(const Path & path); - - bool hasSubstitutes(const Path & path); - Hash queryPathHash(const Path & path); Path queryStatePathDrv(const Path & statePath); @@ -49,6 +45,10 @@ public: Path queryDeriver(const Path & path); + PathSet querySubstitutablePaths(); + + bool hasSubstitutes(const Path & path); + Path addToStore(const Path & srcPath, bool fixed = false, bool recursive = false, string hashAlgo = "", PathFilter & filter = defaultPathFilter); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 16eaa0eee..4c8f85809 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,7 +8,8 @@ namespace nix { bool StoreAPI::hasSubstitutes(const Path & path) { - return !querySubstitutes(path).empty(); + PathSet paths = querySubstitutablePaths(); + return paths.find(path) != paths.end(); } @@ -207,6 +208,45 @@ Path computeStorePathForText(const string & suffix, const string & s, return makeStorePath(type, hash, suffix); } +//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +ValidPathInfo decodeValidPathInfo(std::istream & str) +{ + ValidPathInfo info; + + getline(str, info.path); + if (str.eof()) { info.path = ""; return info; } + + getline(str, info.deriver); + + string s; int n; + + getline(str, s); + if (!string2Int(s, n)) + throw Error("number expected"); + while (n--) { + getline(str, s); + info.references.insert(s); + } + + getline(str, s); + if (!string2Int(s, n)) + throw Error("number expected"); + while (n--) { + getline(str, s); + info.stateReferences.insert(s); + } + + unsigned int u; + getline(str, s); + if (!string2UnsignedInt(s, u)) + throw Error("number expected"); + info.revision = u; + + if (!str || str.eof()) throw Error("missing input"); + return info; +} +//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 817dd8b22..626545c74 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -15,28 +15,6 @@ namespace nix { -/* A substitute is a program invocation that constructs some store - path (typically by fetching it from somewhere, e.g., from the - network). */ -struct Substitute -{ - /* The derivation that built this store path (empty if none). */ - Path deriver; - - /* Program to be executed to create the store path. Must be in - the output path of `storeExpr'. */ - Path program; - - /* Extra arguments to be passed to the program (the first argument - is the store path to be substituted). */ - Strings args; - - bool operator == (const Substitute & sub) const; -}; - -typedef list Substitutes; - - typedef std::map Roots; @@ -65,13 +43,6 @@ public: /* TODO */ virtual bool isValidComponentOrStatePath(const Path & path) = 0; - /* Return the substitutes for the given path. */ - virtual Substitutes querySubstitutes(const Path & path) = 0; - - /* More efficient variant if we just want to know if a path has - substitutes. */ - virtual bool hasSubstitutes(const Path & path); - /* Queries the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; @@ -100,6 +71,13 @@ public: no deriver has been set. */ virtual Path queryDeriver(const Path & path) = 0; + /* Query the set of substitutable paths. */ + virtual PathSet querySubstitutablePaths() = 0; + + /* More efficient variant if we just want to know if a path has + substitutes. */ + virtual bool hasSubstitutes(const Path & path); + /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. If `fixed' is true, then the output of a fixed-output @@ -128,10 +106,10 @@ public: /* Ensure that the output paths of the derivation are valid. If they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths have - substitutes, then those can be used. Second, the output paths - can be created by running the builder, after recursively - building any sub-derivations. */ + can be reached in two ways. First, if the output paths is + substitutable, then build the path that way. Second, the + output paths can be created by running the builder, after + recursively building any sub-derivations. */ virtual void buildDerivations(const PathSet & drvPaths) = 0; /* Ensure that a path is valid. If it is not currently valid, it @@ -339,6 +317,29 @@ extern boost::shared_ptr store; boost::shared_ptr openStore(bool reserveSpace = true); + +/* OLD TODO REMOVE +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; +}; +*/ +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; + PathSet stateReferences; + int unsigned revision; +}; + +ValidPathInfo decodeValidPathInfo(std::istream & str); + + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 995fa1341..497ab4bd6 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -14,7 +14,6 @@ typedef enum { wopIsValidPath, wopIsValidStatePath, wopIsValidComponentOrStatePath, - wopQuerySubstitutes, wopHasSubstitutes, wopQueryPathHash, wopQueryStatePathDrv, diff --git a/src/libutil/types.hh b/src/libutil/types.hh index ad93388d9..5af12f6b3 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -21,23 +21,18 @@ using std::vector; using boost::format; -class Error : public std::exception +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception { protected: string err; public: - Error(const format & f); - ~Error() throw () { }; + BaseError(const format & f); + ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } const string & msg() const throw () { return err; } - Error & addPrefix(const format & f); -}; - -class SysError : public Error -{ -public: - int errNo; - SysError(const format & f); + BaseError & addPrefix(const format & f); }; #define MakeError(newClass, superClass) \ @@ -47,6 +42,15 @@ public: newClass(const format & f) : superClass(f) { }; \ }; +MakeError(Error, BaseError) + +class SysError : public Error +{ +public: + int errNo; + SysError(const format & f); +}; + typedef list Strings; typedef list StringsList; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index effb0174b..4f7effb23 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -24,13 +24,13 @@ extern char * * environ; namespace nix { -Error::Error(const format & f) +BaseError::BaseError(const format & f) { err = f.str(); } -Error & Error::addPrefix(const format & f) +BaseError & BaseError::addPrefix(const format & f) { err = f.str() + err; return *this; @@ -494,6 +494,7 @@ string drainFD(int fd) string result; unsigned char buffer[4096]; while (1) { + checkInterrupt(); ssize_t rd = read(fd, buffer, sizeof buffer); if (rd == -1) { if (errno != EINTR) @@ -777,6 +778,8 @@ void killUser(uid_t uid) string runProgram(Path program, bool searchPath, const Strings & args) { + checkInterrupt(); + /* Create a pipe. */ Pipe pipe; pipe.create(); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c0dbbeb5b..315c09083 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -265,7 +265,7 @@ void inline checkInterrupt() if (_isInterrupted) _interrupted(); } -MakeError(Interrupted, Error) +MakeError(Interrupted, BaseError) /* String packing / unpacking. */ diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index 4f7039c12..c43a9fcd3 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -11,8 +11,6 @@ Operations: --query / -q: query information --read-log / -l: print build log of given store paths - --register-substitutes: register a substitute expression (dangerous!) - --clear-substitutes: clear all substitutes --register-validity: register path validity (dangerous!) --check-validity: check path validity diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 440dcd5f7..f74a030ae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -417,59 +417,6 @@ static void opReadLog(Strings opFlags, Strings opArgs) } } - -static void opRegisterSubstitutes(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - Transaction txn; - createStoreTransaction(txn); - - while (1) { - Path srcPath; - Substitute sub; - PathSet references; - - //TODO TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - PathSet stateReferences; - - getline(cin, srcPath); - if (cin.eof()) break; - getline(cin, sub.deriver); - getline(cin, sub.program); - string s; int n; - getline(cin, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(cin, s); - sub.args.push_back(s); - } - getline(cin, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(cin, s); - references.insert(s); - } - if (!cin || cin.eof()) throw Error("missing input"); - registerSubstitute(txn, srcPath, sub); - setReferences(txn, srcPath, references, stateReferences, 1); //state revision 1, e.g. first commit - } - - txn.commit(); -} - - -static void opClearSubstitutes(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - - clearSubstitutes(); -} - - static void opRegisterValidity(Strings opFlags, Strings opArgs) { bool reregister = false; // !!! maybe this should be the default @@ -484,18 +431,8 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs) ValidPathInfos infos; while (1) { - ValidPathInfo info; - getline(cin, info.path); - if (cin.eof()) break; - getline(cin, info.deriver); - string s; int n; - getline(cin, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(cin, s); - info.references.insert(s); - } - if (!cin || cin.eof()) throw Error("missing input"); + ValidPathInfo info = decodeValidPathInfo(cin); + if (info.path == "") break; if (!store->isValidPath(info.path) || reregister) { /* !!! races */ canonicalisePathMetaData(info.path); @@ -708,10 +645,6 @@ void run(Strings args) op = opQuery; else if (arg == "--read-log" || arg == "-l") op = opReadLog; - else if (arg == "--register-substitutes") - op = opRegisterSubstitutes; - else if (arg == "--clear-substitutes") - op = opClearSubstitutes; else if (arg == "--register-validity") op = opRegisterValidity; else if (arg == "--check-validity")