nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-20 17:29:36 +01:00
Code Activity

Merge branch 'master' of github.com:NixOS/nix into ca-drv

Browse source
This commit is contained in:
Carlo Nucera 2020-07-28 15:36:45 -04:00
commit cfe96bd7ce
199 changed files with 10207 additions and 3179 deletions
Download patch file Download diff file Expand all files Collapse all files

134
src/libstore/binary-cache-store.cc
View file

@ -15,6 +15,7 @@
#include <chrono>
#include <future>
#include <regex>
#include <fstream>
#include <nlohmann/json.hpp>
@ -57,6 +58,13 @@ void BinaryCacheStore::init()
}
}
void BinaryCacheStore::upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType)
{
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
}
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
{
@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
AutoCloseFD openFile(const Path & path)
{
// FIXME: See if we can use the original source to reduce memory usage.
auto nar = make_ref<std::string>(narSource.drain());
auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening file '%1%'", path);
return fd;
}
if (!repair && isValidPath(info.path)) return;
struct FileSource : FdSource
{
AutoCloseFD fd2;
FileSource(const Path & path)
: fd2(openFile(path))
{
fd = fd2.get();
}
};
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
assert(info.narHash && info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
auto [fdTemp, fnTemp] = createTempFile();
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink(htSHA256);
std::shared_ptr<FSAccessor> narAccessor;
{
FdSink fileSink(fdTemp.get());
TeeSink teeSink(fileSink, fileHashSink);
auto compressionSink = makeCompressionSink(compression, teeSink);
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
}
auto now2 = std::chrono::steady_clock::now();
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = info.narSize;
narInfo->narHash = info.narHash;
narInfo->compression = compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), info.narSize,
((1.0 - (double) fileSize / info.narSize) * 100.0),
duration);
/* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */
@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
printStorePath(info.path), printStorePath(ref));
}
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
{
auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true);
listNar(res, ref<FSAccessor>(narAccessor), "", true);
}
}
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), narInfo->narSize,
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
duration);
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
/* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */
@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
upsertFile(narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
"application/x-nix-nar");
} else
stats.narWriteAverted++;
stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteBytes += info.narSize;
stats.narWriteCompressedBytes += fileSize;
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
@ -338,7 +372,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
Hash h;
std::optional<Hash> h;
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
@ -348,10 +382,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s);
}
ValidPathInfo info(makeFixedOutputPath(method, h, name));
ValidPathInfo info(makeFixedOutputPath(method, *h, name));
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
addToStore(info, source, repair, CheckSigs);
return std::move(info.path);
}
@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
addToStore(info, source, repair, CheckSigs);
}
return std::move(info.path);

9
src/libstore/binary-cache-store.hh
View file

@ -36,9 +36,13 @@ public:
virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path,
const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) = 0;
void upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType);
/* Note: subclasses must implement at least one of the two
following getFile() methods. */
@ -75,8 +79,7 @@ public:
{ unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,

109
src/libstore/build.cc
View file

@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
{
this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
state = &DerivationGoal::haveDerivation;
name = fmt("building of %s", worker.store.showPaths(drv.outputPaths()));
name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store)));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation()
retrySubstitution = false;
for (auto & i : drv->outputs)
worker.store.addTempRoot(i.second.path);
worker.store.addTempRoot(i.second.path(worker.store, drv->name));
/* Check what outputs paths are not already valid. */
auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure()
StorePathSet outputClosure;
for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue;
worker.store.computeFSClosure(i.second.path, outputClosure);
worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
for (auto & i : drv->outputs)
outputClosure.erase(i.second.path);
outputClosure.erase(i.second.path(worker.store, drv->name));
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure()
if (i.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(i);
for (auto & j : drv.outputs)
outputsToDrv.insert_or_assign(j.second.path, i);
outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
}
/* Check each path (slow!). */
@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised()
for (auto & j : i.second) {
auto k = inDrv.outputs.find(j);
if (k != inDrv.outputs.end())
worker.store.computeFSClosure(k->second.path, inputPaths);
worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths);
else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild()
goal can start a build, and if not, the main loop will sleep a
few seconds and then retry this goal. */
PathSet lockFiles;
for (auto & outPath : drv->outputPaths())
for (auto & outPath : drv->outputPaths(worker.store))
lockFiles.insert(worker.store.Store::toRealPath(outPath));
if (!outputLocks.lockPaths(lockFiles, "", false)) {
@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild()
return;
}
missingPaths = drv->outputPaths();
missingPaths = drv->outputPaths(worker.store);
if (buildMode != bmCheck)
for (auto & i : validPaths) missingPaths.erase(i);
/* If any of the outputs already exist but are not valid, delete
them. */
for (auto & i : drv->outputs) {
if (worker.store.isValidPath(i.second.path)) continue;
debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
deletePath(worker.store.Store::toRealPath(i.second.path));
if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
}
/* Don't do a remote build if the derivation has the attribute
@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone()
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{worker.store.printStorePath(drvPath)});
PushActivity pact(act.id);
auto outputPaths = drv->outputPaths();
auto outputPaths = drv->outputPaths(worker.store);
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j);
for (auto & k : drv.outputs)
worker.store.computeFSClosure(k.second.path, paths);
worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
}
}
@ -1950,8 +1950,11 @@ void linkOrCopy(const Path & from, const Path & to)
/* Hard-linking fails if we exceed the maximum link count on a
file (e.g. 32000 of ext3), which is quite possible after a
'nix-store --optimise'. FIXME: actually, why don't we just
bind-mount in this case? */
if (errno != EMLINK)
bind-mount in this case?
It can also fail with EPERM in BeegFS v7 and earlier versions
which don't allow hard-links to other directories */
if (errno != EMLINK && errno != EPERM)
throw SysError("linking '%s' to '%s'", to, from);
copyPath(from, to);
}
@ -2012,7 +2015,7 @@ void DerivationGoal::startBuilder()
/* Substitute output placeholders with the actual output paths. */
for (auto & output : drv->outputs)
inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path);
inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
/* Construct the environment passed to the builder. */
initEnv();
@ -2038,7 +2041,10 @@ void DerivationGoal::startBuilder()
if (!std::regex_match(fileName, regex))
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
auto storePath = worker.store.parseStorePath(*i++);
auto storePathS = *i++;
if (!worker.store.isInStore(storePathS))
throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
auto storePath = worker.store.toStorePath(storePathS).first;
/* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName,
@ -2077,7 +2083,7 @@ void DerivationGoal::startBuilder()
for (auto & i : dirsInChroot)
try {
if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
} catch (InvalidPath & e) {
} catch (Error & e) {
throw Error("while processing 'sandbox-paths': %s", e.what());
@ -2194,7 +2200,7 @@ void DerivationGoal::startBuilder()
(typically the dependencies of /bin/sh). Throw them
out. */
for (auto & i : drv->outputs)
dirsInChroot.erase(worker.store.printStorePath(i.second.path));
dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
#elif __APPLE__
/* We don't really have any parent prep work to do (yet?)
@ -2607,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs()
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv->outputs)
outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites);
outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
@ -2750,8 +2756,8 @@ struct RestrictedStore : public LocalFSStore
void queryReferrers(const StorePath & path, StorePathSet & referrers) override
{ }
StorePathSet queryDerivationOutputs(const StorePath & path) override
{ throw Error("queryDerivationOutputs"); }
OutputPathMap queryDerivationOutputMap(const StorePath & path) override
{ throw Error("queryDerivationOutputMap"); }
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ throw Error("queryPathFromHashPart"); }
@ -2762,14 +2768,13 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) override
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
{
next->addToStore(info, narSource, repair, checkSigs, accessor);
next->addToStore(info, narSource, repair, checkSigs);
goal.addDependency(info.path);
}
StorePath addToStoreFromDump(const string & dump, const string & name,
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
@ -2812,7 +2817,7 @@ struct RestrictedStore : public LocalFSStore
auto drv = derivationFromPath(path.path);
for (auto & output : drv.outputs)
if (wantOutput(output.first, path.outputs))
newPaths.insert(output.second.path);
newPaths.insert(output.second.path(*this, drv.name));
} else if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
}
@ -3574,7 +3579,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
if (store.isStorePath(i))
result.insert(store.parseStorePath(i));
else if (drv.outputs.count(i))
result.insert(drv.outputs.find(i)->second.path);
result.insert(drv.outputs.find(i)->second.path(store, drv.name));
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
}
return result;
@ -3612,7 +3617,7 @@ void DerivationGoal::registerOutputs()
if (hook) {
bool allValid = true;
for (auto & i : drv->outputs)
if (!worker.store.isValidPath(i.second.path)) allValid = false;
if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
if (allValid) return;
}
@ -3633,23 +3638,23 @@ void DerivationGoal::registerOutputs()
Nix calls. */
StorePathSet referenceablePaths;
for (auto & p : inputPaths) referenceablePaths.insert(p);
for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path);
for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
for (auto & p : addedPaths) referenceablePaths.insert(p);
/* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all
output paths read-only. */
for (auto & i : drv->outputs) {
auto path = worker.store.printStorePath(i.second.path);
if (!missingPaths.count(i.second.path)) continue;
auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
Path actualPath = path;
if (needsHashRewrite()) {
auto r = redirectedOutputs.find(i.second.path);
auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
if (r != redirectedOutputs.end()) {
auto redirected = worker.store.Store::toRealPath(r->second);
if (buildMode == bmRepair
&& redirectedBadOutputs.count(i.second.path)
&& redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
&& pathExists(redirected))
replaceValidPath(path, redirected);
if (buildMode == bmCheck)
@ -3718,7 +3723,9 @@ void DerivationGoal::registerOutputs()
if (derivationIsFixed(derivationType)) {
if (i.second.hash->method == FileIngestionMethod::Flat) {
FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash;
if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@ -3729,13 +3736,13 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
? hashPath(*i.second.hash->hash.type, actualPath).first
: hashFile(*i.second.hash->hash.type, actualPath);
Hash h2 = outputHash.method == FileIngestionMethod::Recursive
? hashPath(outputHash.hash.type, actualPath).first
: hashFile(outputHash.hash.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
if (i.second.hash->hash != h2) {
if (outputHash.hash != h2) {
/* Throw an error after registering the path as
valid. */
@ -3743,7 +3750,7 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest),
i.second.hash->hash.to_string(SRI, true),
outputHash.hash.to_string(SRI, true),
h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest);
@ -3765,7 +3772,7 @@ void DerivationGoal::registerOutputs()
assert(worker.store.parseStorePath(path) == dest);
ca = FixedOutputHash {
.method = i.second.hash->method,
.method = outputHash.method,
.hash = h2,
};
}
@ -3780,8 +3787,10 @@ void DerivationGoal::registerOutputs()
time. The hash is stored in the database so that we can
verify later on whether nobody has messed with the store. */
debug("scanning for references inside '%1%'", path);
HashResult hash;
auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash));
// HashResult hash;
auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths));
auto references = worker.store.parseStorePathSet(pathSetAndHash.first);
HashResult hash = pathSetAndHash.second;
if (buildMode == bmCheck) {
if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue;
@ -3889,7 +3898,7 @@ void DerivationGoal::registerOutputs()
/* If this is the first round of several, then move the output out of the way. */
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
for (auto & i : drv->outputs) {
auto path = worker.store.printStorePath(i.second.path);
auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
@ -3907,7 +3916,7 @@ void DerivationGoal::registerOutputs()
if the result was not determistic? */
if (curRound == nrRounds) {
for (auto & i : drv->outputs) {
Path prev = worker.store.printStorePath(i.second.path) + checkSuffix;
Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
deletePath(prev);
}
}
@ -4208,9 +4217,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue;
bool good =
worker.store.isValidPath(i.second.path) &&
(!checkHash || worker.pathContentsGood(i.second.path));
if (good == returnValid) result.insert(i.second.path);
worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
(!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
}
return result;
}
@ -5003,7 +5012,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}

2
src/libstore/builtins/buildenv.hh
View file

@ -9,7 +9,7 @@ struct Package {
Path path;
bool active;
int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
};
typedef std::vector<Package> Packages;

26
src/libstore/builtins/fetchurl.cc
View file

@ -58,18 +58,24 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
/* We always have one output, and if it's a fixed-output derivation (as
checked below) it must be the only output */
auto & output = drv.outputs.begin()->second;
/* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat")
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
if (auto hash = std::get_if<DerivationOutputFixed>(&output.output)) {
if (hash->hash.method == FileIngestionMethod::Flat) {
for (auto hashedMirror : settings.hashedMirrors.get()) {
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
fetch(hashedMirror + printHashType(hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
}
}
}
}
/* Otherwise try the specified URL. */
fetch(mainUrl);

16
src/libstore/content-address.cc
View file

@ -3,7 +3,7 @@
namespace nix {
std::string FixedOutputHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
Hash hash = Hash(string(hashTypeAndHash));
if (*hash.type != htSHA256) {
if (hash.type != htSHA256) {
throw Error("parseContentAddress: the text hash should have type SHA256");
}
return TextHash { hash };
@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : "";
}
Hash getContentAddressHash(const ContentAddress & ca)
{
return std::visit(overloaded {
[](TextHash th) {
return th.hash;
},
[](FixedOutputHash fsh) {
return fsh.hash;
}
}, ca);
}
}

2
src/libstore/content-address.hh
View file

@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
}

86
src/libstore/daemon.cc
View file

@ -78,10 +78,10 @@ struct TunnelLogger : public Logger
if (ei.level > verbosity) return;
std::stringstream oss;
oss << ei;
showErrorInfo(oss, ei, false);
StringSink buf;
buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
buf << STDERR_NEXT << oss.str();
enqueueMsg(*buf.s);
}
@ -173,31 +173,6 @@ struct TunnelSource : BufferedSource
}
};
/* If the NAR archive contains a single file at top-level, then save
the contents of the file to `s'. Otherwise barf. */
struct RetrieveRegularNARSink : ParseSink
{
bool regular;
string s;
RetrieveRegularNARSink() : regular(true) { }
void createDirectory(const Path & path)
{
regular = false;
}
void receiveContents(unsigned char * data, unsigned int len)
{
s.append((const char *) data, len);
}
void createSymlink(const Path & path, const string & target)
{
regular = false;
}
};
struct ClientSettings
{
bool keepFailed;
@ -314,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
to << hash.to_string(Base16, false);
to << hash->to_string(Base16, false);
break;
}
@ -347,6 +322,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case wopQueryDerivationOutputMap: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
OutputPathMap outputs = store->queryDerivationOutputMap(path);
logger->stopWork();
writeOutputPathMap(*store, to, outputs);
break;
}
case wopQueryDeriver: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
@ -366,42 +350,44 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopAddToStore: {
std::string s, baseName;
HashType hashAlgo;
std::string baseName;
FileIngestionMethod method;
{
bool fixed; uint8_t recursive;
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
bool fixed;
uint8_t recursive;
std::string hashAlgoRaw;
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
hashAlgoRaw = "sha256";
method = FileIngestionMethod::Recursive;
}
hashAlgo = parseHashType(hashAlgoRaw);
}
HashType hashAlgo = parseHashType(s);
TeeSource savedNAR(from);
RetrieveRegularNARSink savedRegular;
StringSink saved;
TeeSource savedNARSource(from, saved);
RetrieveRegularNARSink savedRegular { saved };
if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNAR);
parseDump(sink, savedNARSource);
} else
parseDump(savedRegular, from);
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
baseName,
method,
hashAlgo);
// FIXME: try to stream directly from `from`.
StringSource dumpSource { *saved.s };
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
logger->stopWork();
to << store->printStorePath(path);
@ -433,7 +419,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopImportPaths: {
logger->startWork();
TunnelSource source(from, to);
auto paths = store->importPaths(source, nullptr,
auto paths = store->importPaths(source,
trusted ? NoCheckSigs : CheckSigs);
logger->stopWork();
Strings paths2;
@ -465,7 +451,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopBuildDerivation: {
auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv;
readDerivation(from, *store, drv);
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
if (!trusted)
@ -646,7 +632,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash.to_string(Base16, false);
<< info->narHash->to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
@ -717,22 +703,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!trusted)
info.ultimate = false;
std::string saved;
std::unique_ptr<Source> source;
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to);
else {
TeeSink tee(from);
parseDump(tee, tee.source);
saved = std::move(*tee.source.data);
source = std::make_unique<StringSource>(saved);
StringSink saved;
TeeSource tee { from, saved };
ParseSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(std::move(*saved.s));
}
logger->startWork();
// FIXME: race if addToStore doesn't read source?
store->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
dontCheckSigs ? NoCheckSigs : CheckSigs);
logger->stopWork();
break;

135
src/libstore/derivations.cc
View file

@ -4,7 +4,6 @@
#include "util.hh"
#include "worker-protocol.hh"
#include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix {
@ -34,12 +33,20 @@ bool derivationIsImpure(DerivationType dt) {
abort();
}
const StorePath & BasicDerivation::findOutput(const string & id) const
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
{
auto i = outputs.find(id);
if (i == outputs.end())
throw Error("derivation has no output '%s'", id);
return i->second.path;
return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) {
return doi.path;
},
[&](DerivationOutputFixed dof) {
return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
}
}, output);
}
@ -127,14 +134,13 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
@ -142,23 +148,30 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
return DerivationOutput {
.output = DerivationOutputFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
},
}
};
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path),
}
};
}
static Derivation parseDerivation(const Store & store, const string & s)
static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{
Derivation drv;
istringstream_nocopy str(s);
drv.name = name;
std::istringstream str(std::move(s));
expect(str, "Derive([");
/* Parse the list of outputs. */
@ -201,10 +214,10 @@ static Derivation parseDerivation(const Store & store, const string & s)
}
Derivation readDerivation(const Store & store, const Path & drvPath)
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
{
try {
return parseDerivation(store, readFile(drvPath));
return parseDerivation(store, readFile(drvPath), name);
} catch (FormatError & e) {
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
}
@ -222,7 +235,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor();
try {
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
}
@ -290,10 +303,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
s += ','; printUnquotedString(s,
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
} else {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
s += ')';
}
@ -349,7 +366,7 @@ DerivationType BasicDerivation::type() const
{
if (outputs.size() == 1 &&
outputs.begin()->first == "out" &&
outputs.begin()->second.hash)
std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output))
{
return DerivationType::CAFixed;
} else {
@ -387,10 +404,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
switch (drv.type()) {
case DerivationType::CAFixed: {
DerivationOutputs::const_iterator i = drv.outputs.begin();
auto hash = std::get<DerivationOutputFixed>(i->second.output);
return hashString(htSHA256, "fixed:out:"
+ i->second.hash->printMethodAlgo() + ":"
+ i->second.hash->hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path));
+ hash.hash.printMethodAlgo() + ":"
+ hash.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path(store, drv.name)));
}
default:
break;
@ -427,11 +445,11 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
}
StorePathSet BasicDerivation::outputPaths() const
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputs)
paths.insert(i.second.path);
paths.insert(i.second.path(store, name));
return paths;
}
@ -439,26 +457,29 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
const auto hash = readString(in);
auto hash = readString(in);
std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
auto hashType = parseHashType(hashAlgo);
return DerivationOutput {
.output = DerivationOutputFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
},
}
};
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path),
}
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
StringSet BasicDerivation::outputNames() const
@ -470,8 +491,19 @@ StringSet BasicDerivation::outputNames() const
}
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension));
nameWithSuffix.remove_suffix(extension.size());
return nameWithSuffix;
}
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
{
drv.name = name;
drv.outputs.clear();
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
@ -498,11 +530,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
for (auto & i : drv.outputs)
for (auto & i : drv.outputs) {
out << i.first
<< store.printStorePath(i.second.path)
<< i.second.hash->printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false);
<< store.printStorePath(i.second.path(store, drv.name));
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
out << hash->hash.printMethodAlgo()
<< hash->hash.hash.to_string(Base16, false);
} else {
out << "" << "";
}
}
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();

27
src/libstore/derivations.hh
View file

@ -13,10 +13,20 @@ namespace nix {
/* Abstract syntax of derivations. */
struct DerivationOutput
struct DerivationOutputInputAddressed
{
StorePath path;
std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
};
struct DerivationOutputFixed
{
FixedOutputHash hash; /* hash used for expected hash computation */
};
struct DerivationOutput
{
std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output;
StorePath path(const Store & store, std::string_view drvName) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -53,24 +63,23 @@ struct BasicDerivation
Path builder;
Strings args;
StringPairs env;
std::string name;
BasicDerivation() { }
virtual ~BasicDerivation() { };
/* Return the path corresponding to the output identifier `id' in
the given derivation. */
const StorePath & findOutput(const std::string & id) const;
bool isBuiltin() const;
/* Return true iff this is a fixed-output derivation. */
DerivationType type() const;
/* Return the output paths of a derivation. */
StorePathSet outputPaths() const;
StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
static std::string_view nameFromPath(const StorePath & storePath);
};
struct Derivation : BasicDerivation
@ -94,7 +103,7 @@ StorePath writeDerivation(ref<Store> store,
const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */
Derivation readDerivation(const Store & store, const Path & drvPath);
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
// FIXME: remove
bool isDerivation(const string & fileName);
@ -111,7 +120,7 @@ bool wantOutput(const string & output, const std::set<string> & wanted);
struct Source;
struct Sink;
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv);
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
std::string hashPlaceholder(const std::string & outputName);

51
src/libstore/export-import.cc
View file

@ -7,24 +7,6 @@
namespace nix {
struct HashAndWriteSink : Sink
{
Sink & writeSink;
HashSink hashSink;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
}
virtual void operator () (const unsigned char * data, size_t len)
{
writeSink(data, len);
hashSink(data, len);
}
Hash currentHash()
{
return hashSink.currentHash().first;
}
};
void Store::exportPaths(const StorePathSet & paths, Sink & sink)
{
auto sorted = topoSortPaths(paths);
@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
{
auto info = queryPathInfo(path);
HashAndWriteSink hashAndWriteSink(sink);
HashSink hashSink(htSHA256);
TeeSink teeSink(sink, hashSink);
narFromPath(path, hashAndWriteSink);
narFromPath(path, teeSink);
/* Refuse to export paths that have changed. This prevents
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
hashAndWriteSink
teeSink
<< exportMagic
<< printStorePath(path);
writeStorePaths(*this, hashAndWriteSink, info->references);
hashAndWriteSink
writeStorePaths(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
}
StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
{
StorePaths res;
while (true) {
@ -77,8 +60,10 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */
TeeSink tee(source);
parseDump(tee, tee.source);
StringSink saved;
TeeSource tee { source, saved };
ParseSink ether;
parseDump(ether, tee);
uint32_t magic = readInt(source);
if (magic != exportMagic)
@ -94,16 +79,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *tee.source.data);
info.narSize = tee.source.data->size();
info.narHash = hashString(htSHA256, *saved.s);
info.narSize = saved.s->size();
// Ignore optional legacy signature.
if (readInt(source) == 1)
readString(source);
// Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.source.data };
addToStore(info, source, NoRepair, checkSigs, accessor);
auto source = StringSource { *saved.s };
addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path);
}

57
src/libstore/filetransfer.cc
View file

@ -22,6 +22,7 @@
#include <queue>
#include <random>
#include <thread>
#include <regex>
using namespace std::string_literals;
@ -56,7 +57,7 @@ struct curlFileTransfer : public FileTransfer
Callback<FileTransferResult> callback;
CURL * req = 0;
bool active = false; // whether the handle has been added to the multi object
std::string status;
std::string statusMsg;
unsigned int attempt = 0;
@ -123,7 +124,7 @@ struct curlFileTransfer : public FileTransfer
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
} catch (...) {
ignoreException();
}
@ -144,6 +145,7 @@ struct curlFileTransfer : public FileTransfer
LambdaSink finalSink;
std::shared_ptr<CompressionSink> decompressionSink;
std::optional<StringSink> errorSink;
std::exception_ptr writeException;
@ -153,9 +155,19 @@ struct curlFileTransfer : public FileTransfer
size_t realSize = size * nmemb;
result.bodySize += realSize;
if (!decompressionSink)
if (!decompressionSink) {
decompressionSink = makeDecompressionSink(encoding, finalSink);
if (! successfulStatuses.count(getHTTPStatus())) {
// In this case we want to construct a TeeSink, to keep
// the response around (which we figure won't be big
// like an actual download should be) to improve error
// messages.
errorSink = StringSink { };
}
}
if (errorSink)
(*errorSink)((unsigned char *) contents, realSize);
(*decompressionSink)((unsigned char *) contents, realSize);
return realSize;
@ -175,12 +187,13 @@ struct curlFileTransfer : public FileTransfer
size_t realSize = size * nmemb;
std::string line((char *) contents, realSize);
printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
std::smatch match;
if (std::regex_match(line, match, statusLine)) {
result.etag = "";
auto ss = tokenizeString<vector<string>>(line, " ");
status = ss.size() >= 2 ? ss[1] : "";
result.data = std::make_shared<std::string>();
result.bodySize = 0;
statusMsg = trim(match[1]);
acceptRanges = false;
encoding = "";
} else {
@ -194,7 +207,9 @@ struct curlFileTransfer : public FileTransfer
the expected ETag on a 200 response, then shut
down the connection because we already have the
data. */
if (result.etag == request.expectedETag && status == "200") {
long httpStatus = 0;
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
if (result.etag == request.expectedETag && httpStatus == 200) {
debug(format("shutting down on 200 HTTP response with expected ETag"));
return 0;
}
@ -408,16 +423,21 @@ struct curlFileTransfer : public FileTransfer
attempt++;
std::shared_ptr<std::string> response;
if (errorSink)
response = errorSink->s;
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
: httpStatus != 0
? FileTransferError(err,
fmt("unable to %s '%s': HTTP error %d",
request.verb(), request.uri, httpStatus)
response,
fmt("unable to %s '%s': HTTP error %d ('%s')",
request.verb(), request.uri, httpStatus, statusMsg)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
: FileTransferError(err,
response,
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
@ -675,7 +695,7 @@ struct curlFileTransfer : public FileTransfer
auto s3Res = s3Helper.getObject(bucketName, key);
FileTransferResult res;
if (!s3Res.data)
throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
res.data = s3Res.data;
callback(std::move(res));
#else
@ -820,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
}
}
template<typename... Args>
FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
: Error(args...), error(error), response(response)
{
const auto hf = hintfmt(args...);
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
} else {
err.hint = hf;
}
}
bool isUri(const string & s)
{
if (s.compare(0, 8, "channel:") == 0) return true;

8
src/libstore/filetransfer.hh
View file

@ -103,10 +103,12 @@ class FileTransferError : public Error
{
public:
FileTransfer::Error error;
std::shared_ptr<string> response; // intentionally optional
template<typename... Args>
FileTransferError(FileTransfer::Error error, const Args & ... args)
: Error(args...), error(error)
{ }
FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
virtual const char* sname() const override { return "FileTransferError"; }
};
bool isUri(const string & s);

30
src/libstore/gc.cc
View file

@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{
auto foundRoot = [&](const Path & path, const Path & target) {
auto storePath = maybeParseStorePath(toStorePath(target));
if (storePath && isValidPath(*storePath))
roots[std::move(*storePath)].emplace(path);
else
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
try {
auto storePath = toStorePath(target).first;
if (isValidPath(storePath))
roots[std::move(storePath)].emplace(path);
else
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
} catch (BadStorePath &) { }
};
try {
@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue;
Path pathS = toStorePath(target);
if (!isStorePath(pathS)) continue;
auto path = parseStorePath(pathS);
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", pathS);
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
try {
auto path = toStorePath(target).first;
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", printStorePath(path));
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
} catch (BadStorePath &) { }
}
}

2
src/libstore/globals.cc
View file

@ -35,7 +35,7 @@ Settings::Settings()
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
, nixManDir(canonPath(NIX_MAN_DIR))
, nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
, nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
{
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";

10
src/libstore/globals.hh
View file

@ -196,10 +196,6 @@ public:
/* Whether to lock the Nix client and worker to the same CPU. */
bool lockCPU;
/* Whether to show a stack trace if Nix evaluation fails. */
Setting<bool> showTrace{this, false, "show-trace",
"Whether to show a stack trace on evaluation errors."};
Setting<SandboxMode> sandboxMode{this,
#if __linux__
smEnabled
@ -369,6 +365,12 @@ public:
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
"Maximum size of NARs before spilling them to disk."};
Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
"Path or URI of the global flake registry."};
};

4
src/libstore/http-binary-cache-store.cc
View file

@ -100,11 +100,11 @@ protected:
}
void upsertFile(const std::string & path,
const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto req = FileTransferRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.mimeType = mimeType;
try {
getFileTransfer()->upload(req);

7
src/libstore/legacy-ssh-store.cc
View file

@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? Hash() : Hash(s);
info->narHash = s.empty() ? std::optional<Hash>{} : Hash{s};
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}
@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
}
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override
RepairFlag repair, CheckSigsFlag checkSigs) override
{
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
@ -139,7 +138,7 @@ struct LegacySSHStore : public Store
<< cmdAddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
<< info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to
<< info.registrationTime

35
src/libstore/local-binary-cache-store.cc
View file

@ -31,8 +31,18 @@ protected:
bool fileExists(const std::string & path) override;
void upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType) override;
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto path2 = binaryCacheDir + "/" + path;
Path tmp = path2 + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
if (rename(tmp.c_str(), path2.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path2);
del.cancel();
}
void getFile(const std::string & path, Sink & sink) override
{
@ -52,7 +62,9 @@ protected:
if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo"))
continue;
paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
paths.insert(parseStorePath(
storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+ "-" + MissingName));
}
return paths;
@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
BinaryCacheStore::init();
}
static void atomicWrite(const Path & path, const std::string & s)
{
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path);
del.cancel();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path)
{
return pathExists(binaryCacheDir + "/" + path);
}
void LocalBinaryCacheStore::upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType)
{
atomicWrite(binaryCacheDir + "/" + path, data);
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>

6
src/libstore/local-fs-store.cc
View file

@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
Path toRealPath(const Path & path)
{
Path storePath = store->toStorePath(path);
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", storePath);
auto storePath = store->toStorePath(path).first;
if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}

183
src/libstore/local-store.cc
View file

@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
DerivationOutputs::const_iterator out = drv.outputs.find("out");
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
check(
makeFixedOutputPath(
out->second.hash->method,
out->second.hash->hash,
drvName),
out->second.path, "out");
}
else {
Hash h = hashDerivationModulo(*this, drv, true);
for (auto & i : drv.outputs)
check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
}
}
@ -586,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtRegisterValidPath.use()
(printStorePath(info.path))
(info.narHash.to_string(Base16, true))
(info.narHash->to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0)
@ -594,7 +587,7 @@ uint64_t LocalStore::addValidPath(State & state,
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca)
.exec();
uint64_t id = sqlite3_last_insert_rowid(state.db);
uint64_t id = state.db.getLastInsertedRowId();
/* If this is a derivation, then store the derivation outputs in
the database. This is useful for the garbage collector: it can
@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtAddDerivationOutput.use()
(id)
(i.first)
(printStorePath(i.second.path))
(printStorePath(i.second.path(*this, drv.name)))
.exec();
}
}
@ -686,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
state.stmtUpdatePathInfo.use()
(info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16, true))
(info.narHash->to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca)
@ -774,17 +767,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
}
StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
{
return retrySQLite<StorePathSet>([&]() {
return retrySQLite<OutputPathMap>([&]() {
auto state(_state.lock());
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
(queryValidPathId(*state, path)));
StorePathSet outputs;
OutputPathMap outputs;
while (useQueryDerivationOutputs.next())
outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
outputs.emplace(
useQueryDerivationOutputs.getStr(0),
parseStorePath(useQueryDerivationOutputs.getStr(1))
);
return outputs;
});
@ -897,7 +893,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
StorePathSet paths;
for (auto & i : infos) {
assert(i.narHash.type == htSHA256);
assert(i.narHash && i.narHash->type == htSHA256);
if (isValidPath_(*state, i.path))
updatePathInfo(*state, i);
else
@ -959,7 +955,7 @@ const PublicKeys & LocalStore::getPublicKeys()
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
RepairFlag repair, CheckSigsFlag checkSigs)
{
if (!info.narHash)
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@ -973,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
PathLocks outputLock;
Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
auto realPath = Store::toRealPath(info.path);
/* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a
@ -1010,7 +1006,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true));
if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
@ -1030,12 +1026,78 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
}
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
readFile(srcPath, sink);
});
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
}
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{
Hash h = hashString(hashAlgo, dump);
/* For computing the store path. */
auto hashSink = std::make_unique<HashSink>(hashAlgo);
TeeSource source { source0, *hashSink };
auto dstPath = makeFixedOutputPath(method, h, name);
/* Read the source path into memory, but only if it's up to
narBufferSize bytes. If it's larger, write it to a temporary
location in the Nix store. If the subsequently computed
destination store path is already valid, we just delete the
temporary path. Otherwise, we move it to the destination store
path. */
bool inMemory = false;
std::string dump;
/* Fill out buffer, and decide whether we are working strictly in
memory based on whether we break out because the buffer is full
or the original source is empty */
while (dump.size() < settings.narBufferSize) {
auto oldSize = dump.size();
constexpr size_t chunkSize = 65536;
auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
dump.resize(oldSize + want);
auto got = 0;
try {
got = source.read((uint8_t *) dump.data() + oldSize, want);
} catch (EndOfFile &) {
inMemory = true;
break;
}
dump.resize(oldSize + got);
}
std::unique_ptr<AutoDelete> delTempDir;
Path tempPath;
if (!inMemory) {
/* Drain what we pulled so far, and then keep on pulling */
StringSource dumpSource { dump };
ChainSource bothSource { dumpSource, source };
auto tempDir = createTempDir(realStoreDir, "add");
delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x";
if (method == FileIngestionMethod::Recursive)
restorePath(tempPath, bothSource);
else
writeFile(tempPath, bothSource);
dump.clear();
}
auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name);
addTempRoot(dstPath);
@ -1044,8 +1106,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
@ -1055,31 +1116,36 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
autoGC();
if (method == FileIngestionMethod::Recursive) {
StringSource source(dump);
restorePath(realPath, source);
} else
writeFile(realPath, dump);
if (inMemory) {
StringSource dumpSource { dump };
/* Restore from the NAR in memory. */
if (method == FileIngestionMethod::Recursive)
restorePath(realPath, dumpSource);
else
writeFile(realPath, dumpSource);
} else {
/* Move the temporary path we restored above. */
if (rename(tempPath.c_str(), realPath.c_str()))
throw Error("renaming '%s' to '%s'", tempPath, realPath);
}
canonicalisePathMetaData(realPath, -1);
/* For computing the nar hash. In recursive SHA-256 mode, this
is the same as the store hash, so no need to do it again. */
auto narHash = std::pair { hash, size };
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
HashSink narSink { htSHA256 };
dumpPath(realPath, narSink);
narHash = narSink.finish();
}
/* Register the SHA-256 hash of the NAR serialisation of
the path in the database. We may just have computed it
above (if called with recursive == true and hashAlgo ==
sha256); otherwise, compute it here. */
HashResult hash;
if (method == FileIngestionMethod::Recursive) {
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
hash.second = dump.size();
} else
hash = hashPath(htSHA256, realPath);
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
optimisePath(realPath); // FIXME: combine with hashPath()
optimisePath(realPath);
ValidPathInfo info(dstPath);
info.narHash = hash.first;
info.narSize = hash.second;
info.ca = FixedOutputHash { .method = method, .hash = h };
info.narHash = narHash.first;
info.narSize = narHash.second;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info);
}
@ -1090,24 +1156,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
}
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
Path srcPath(absPath(_srcPath));
/* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = make_ref<std::string>(readFile(srcPath));
return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
}
StorePath LocalStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
@ -1118,8 +1166,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(dstPath)) {
Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
@ -1255,9 +1302,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
std::unique_ptr<AbstractHashSink> hashSink;
if (!info->ca || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(*info->narHash.type);
hashSink = std::make_unique<HashSink>(info->narHash->type);
else
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
@ -1266,7 +1313,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
logError({
.name = "Invalid hash - path modified",
.hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true))
});
if (repair) repairPath(i); else errors = true;
} else {

7
src/libstore/local-store.hh
View file

@ -133,7 +133,7 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override;
StorePathSet queryDerivationOutputs(const StorePath & path) override;
OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
@ -143,8 +143,7 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
@ -154,7 +153,7 @@ public:
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
StorePath addToStoreFromDump(const string & dump, const string & name,
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
StorePath addTextToStore(const string & name, const string & s,

3
src/libstore/local.mk
View file

@ -61,3 +61,6 @@ $(d)/build.cc:
clean-files += $(d)/schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))

4
src/libstore/misc.cc
View file

@ -198,8 +198,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
PathSet invalid;
for (auto & j : drv->outputs)
if (wantOutput(j.first, path.outputs)
&& !isValidPath(j.second.path))
invalid.insert(printStorePath(j.second.path));
&& !isValidPath(j.second.path(*this, drv->name)))
invalid.insert(printStorePath(j.second.path(*this, drv->name)));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {

46
src/libstore/nar-accessor.cc
View file

@ -18,7 +18,7 @@ struct NarMember
/* If this is a regular file, position of the contents of this
file in the NAR. */
size_t start = 0, size = 0;
uint64_t start = 0, size = 0;
std::string target;
@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor
NarMember root;
struct NarIndexer : ParseSink, StringSource
struct NarIndexer : ParseSink, Source
{
NarAccessor & acc;
Source & source;
std::stack<NarMember *> parents;
std::string currentStart;
bool isExec = false;
NarIndexer(NarAccessor & acc, const std::string & nar)
: StringSource(nar), acc(acc)
uint64_t pos = 0;
NarIndexer(NarAccessor & acc, Source & source)
: acc(acc), source(source)
{ }
void createMember(const Path & path, NarMember member) {
@ -79,31 +81,38 @@ struct NarAccessor : public FSAccessor
void preallocateContents(unsigned long long size) override
{
currentStart = string(s, pos, 16);
assert(size <= std::numeric_limits<size_t>::max());
parents.top()->size = (size_t)size;
assert(size <= std::numeric_limits<uint64_t>::max());
parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
}
void receiveContents(unsigned char * data, unsigned int len) override
{
// Sanity check
if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char *) data, 16));
currentStart.clear();
}
}
{ }
void createSymlink(const Path & path, const string & target) override
{
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
size_t read(unsigned char * data, size_t len) override
{
auto n = source.read(data, len);
pos += n;
return n;
}
};
NarAccessor(ref<const std::string> nar) : nar(nar)
{
NarIndexer indexer(*this, *nar);
StringSource source(*nar);
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
NarAccessor(Source & source)
{
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
return make_ref<NarAccessor>(nar);
}
ref<FSAccessor> makeNarAccessor(Source & source)
{
return make_ref<NarAccessor>(source);
}
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes)
{

4
src/libstore/nar-accessor.hh
View file

@ -6,10 +6,14 @@
namespace nix {
struct Source;
/* Return an object that provides access to the contents of a NAR
file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
ref<FSAccessor> makeNarAccessor(Source & source);
/* Create a NAR accessor from a NAR listing (in the format produced by
listNar()). The callback getNarBytes(offset, length) is used by the
readFile() method of the accessor to get the contents of files

4
src/libstore/nar-info-disk-cache.cc
View file

@ -230,9 +230,9 @@ public:
(std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string(Base32, true))
(info->narHash->to_string(Base32, true))
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)

27
src/libstore/nar-info.cc
View file

@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
: ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
{
auto corrupt = [&]() {
throw Error("NAR info file '%1%' is corrupt", whence);
return Error("NAR info file '%1%' is corrupt", whence);
};
auto parseHashField = [&](const string & s) {
try {
return Hash(s);
} catch (BadHash &) {
corrupt();
return Hash(); // never reached
throw corrupt();
}
};
@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
while (pos < s.size()) {
size_t colon = s.find(':', pos);
if (colon == std::string::npos) corrupt();
if (colon == std::string::npos) throw corrupt();
std::string name(s, pos, colon - pos);
size_t eol = s.find('\n', colon + 2);
if (eol == std::string::npos) corrupt();
if (eol == std::string::npos) throw corrupt();
std::string value(s, colon + 2, eol - colon - 2);
@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "FileHash")
fileHash = parseHashField(value);
else if (name == "FileSize") {
if (!string2Int(value, fileSize)) corrupt();
if (!string2Int(value, fileSize)) throw corrupt();
}
else if (name == "NarHash")
narHash = parseHashField(value);
else if (name == "NarSize") {
if (!string2Int(value, narSize)) corrupt();
if (!string2Int(value, narSize)) throw corrupt();
}
else if (name == "References") {
auto refs = tokenizeString<Strings>(value, " ");
if (!references.empty()) corrupt();
if (!references.empty()) throw corrupt();
for (auto & r : refs)
references.insert(StorePath(r));
}
@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
if (ca) corrupt();
if (ca) throw corrupt();
// FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value);
}
@ -77,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (compression == "") compression = "bzip2";
if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt();
if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt();
}
std::string NarInfo::to_string(const Store & store) const
@ -87,11 +86,11 @@ std::string NarInfo::to_string(const Store & store) const
res += "URL: " + url + "\n";
assert(compression != "");
res += "Compression: " + compression + "\n";
assert(fileHash.type == htSHA256);
res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
assert(fileHash && fileHash->type == htSHA256);
res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n";
assert(narHash.type == htSHA256);
res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
assert(narHash && narHash->type == htSHA256);
res += "NarHash: " + narHash->to_string(Base32, true) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n";
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";

2
src/libstore/nar-info.hh
View file

@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo
{
std::string url;
std::string compression;
Hash fileHash;
std::optional<Hash> fileHash;
uint64_t fileSize = 0;
std::string system;

2
src/libstore/path.cc
View file

@ -2,8 +2,6 @@
namespace nix {
MakeError(BadStorePath, Error);
static void checkName(std::string_view path, std::string_view name)
{
if (name.empty())

1
src/libstore/path.hh
View file

@ -62,6 +62,7 @@ public:
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";

81
src/libstore/profiles.cc
View file

@ -12,30 +12,24 @@
namespace nix {
static bool cmpGensByNumber(const Generation & a, const Generation & b)
{
return a.number < b.number;
}
/* Parse a generation name of the format
`<profilename>-<number>-link'. */
static int parseName(const string & profileName, const string & name)
static std::optional<GenerationNumber> parseName(const string & profileName, const string & name)
{
if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
if (string(name, 0, profileName.size() + 1) != profileName + "-") return {};
string s = string(name, profileName.size() + 1);
string::size_type p = s.find("-link");
if (p == string::npos) return -1;
int n;
if (p == string::npos) return {};
unsigned int n;
if (string2Int(string(s, 0, p), n) && n >= 0)
return n;
else
return -1;
return {};
}
Generations findGenerations(Path profile, int & curGen)
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile)
{
Generations gens;
@ -43,30 +37,34 @@ Generations findGenerations(Path profile, int & curGen)
auto profileName = std::string(baseNameOf(profile));
for (auto & i : readDirectory(profileDir)) {
int n;
if ((n = parseName(profileName, i.name)) != -1) {
Generation gen;
gen.path = profileDir + "/" + i.name;
gen.number = n;
if (auto n = parseName(profileName, i.name)) {
auto path = profileDir + "/" + i.name;
struct stat st;
if (lstat(gen.path.c_str(), &st) != 0)
throw SysError("statting '%1%'", gen.path);
gen.creationTime = st.st_mtime;
gens.push_back(gen);
if (lstat(path.c_str(), &st) != 0)
throw SysError("statting '%1%'", path);
gens.push_back({
.number = *n,
.path = path,
.creationTime = st.st_mtime
});
}
}
gens.sort(cmpGensByNumber);
gens.sort([](const Generation & a, const Generation & b)
{
return a.number < b.number;
});
curGen = pathExists(profile)
return {
gens,
pathExists(profile)
? parseName(profileName, readLink(profile))
: -1;
return gens;
: std::nullopt
};
}
static void makeName(const Path & profile, unsigned int num,
static void makeName(const Path & profile, GenerationNumber num,
Path & outLink)
{
Path prefix = (format("%1%-%2%") % profile % num).str();
@ -78,10 +76,9 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
{
/* The new generation number should be higher than old the
previous ones. */
int dummy;
Generations gens = findGenerations(profile, dummy);
auto [gens, dummy] = findGenerations(profile);
unsigned int num;
GenerationNumber num;
if (gens.size() > 0) {
Generation last = gens.back();
@ -121,7 +118,7 @@ static void removeFile(const Path & path)
}
void deleteGeneration(const Path & profile, unsigned int gen)
void deleteGeneration(const Path & profile, GenerationNumber gen)
{
Path generation;
makeName(profile, gen, generation);
@ -129,7 +126,7 @@ void deleteGeneration(const Path & profile, unsigned int gen)
}
static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun)
static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
{
if (dryRun)
printInfo(format("would remove generation %1%") % gen);
@ -140,31 +137,29 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu
}
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun)
void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
int curGen;
Generations gens = findGenerations(profile, curGen);
auto [gens, curGen] = findGenerations(profile);
if (gensToDelete.find(curGen) != gensToDelete.end())
if (gensToDelete.count(*curGen))
throw Error("cannot delete current generation of profile %1%'", profile);
for (auto & i : gens) {
if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
if (!gensToDelete.count(i.number)) continue;
deleteGeneration2(profile, i.number, dryRun);
}
}
void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
int curGen;
bool fromCurGen = false;
Generations gens = findGenerations(profile, curGen);
auto [gens, curGen] = findGenerations(profile);
for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
if (i->number == curGen) {
fromCurGen = true;
@ -186,8 +181,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun)
PathLocks lock;
lockProfile(lock, profile);
int curGen;
Generations gens = findGenerations(profile, curGen);
auto [gens, curGen] = findGenerations(profile);
for (auto & i : gens)
if (i.number != curGen)
@ -200,8 +194,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
PathLocks lock;
lockProfile(lock, profile);
int curGen;
Generations gens = findGenerations(profile, curGen);
auto [gens, curGen] = findGenerations(profile);
bool canDelete = false;
for (auto i = gens.rbegin(); i != gens.rend(); ++i)

25
src/libstore/profiles.hh
View file

@ -9,37 +9,32 @@
namespace nix {
typedef unsigned int GenerationNumber;
struct Generation
{
int number;
GenerationNumber number;
Path path;
time_t creationTime;
Generation()
{
number = -1;
}
operator bool() const
{
return number != -1;
}
};
typedef list<Generation> Generations;
typedef std::list<Generation> Generations;
/* Returns the list of currently present generations for the specified
profile, sorted by generation number. */
Generations findGenerations(Path profile, int & curGen);
profile, sorted by generation number. Also returns the number of
the current generation. */
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
class LocalFSStore;
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
void deleteGeneration(const Path & profile, unsigned int gen);
void deleteGeneration(const Path & profile, GenerationNumber gen);
void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun);
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun);
void deleteOldGenerations(const Path & profile, bool dryRun);

21
src/libstore/references.cc
View file

@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len,
struct RefScanSink : Sink
{
HashSink hashSink;
StringSet hashes;
StringSet seen;
string tail;
RefScanSink() : hashSink(htSHA256) { }
RefScanSink() { }
void operator () (const unsigned char * data, size_t len);
};
@ -62,8 +61,6 @@ struct RefScanSink : Sink
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
hashSink(data, len);
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
@ -79,10 +76,12 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
}
PathSet scanForReferences(const string & path,
const PathSet & refs, HashResult & hash)
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
{
RefScanSink sink;
RefScanSink refsSink;
HashSink hashSink { htSHA256 };
TeeSink sink { refsSink, hashSink };
std::map<string, Path> backMap;
/* For efficiency (and a higher hit rate), just search for the
@ -97,7 +96,7 @@ PathSet scanForReferences(const string & path,
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
// parseHash(htSHA256, s);
sink.hashes.insert(s);
refsSink.hashes.insert(s);
backMap[s] = i;
}
@ -106,15 +105,15 @@ PathSet scanForReferences(const string & path,
/* Map the hashes found back to their store paths. */
PathSet found;
for (auto & i : sink.seen) {
for (auto & i : refsSink.seen) {
std::map<string, Path>::iterator j;
if ((j = backMap.find(i)) == backMap.end()) abort();
found.insert(j->second);
}
hash = sink.hashSink.finish();
auto hash = hashSink.finish();
return found;
return std::pair<PathSet, HashResult>(found, hash);
}

3
src/libstore/references.hh
View file

@ -5,8 +5,7 @@
namespace nix {
PathSet scanForReferences(const Path & path, const PathSet & refs,
HashResult & hash);
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
struct RewritingSink : Sink
{

33
src/libstore/remote-fs-accessor.cc
View file

@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
createDirs(cacheDir);
}
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
{
assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor)
{
nars.emplace(storePath, narAccessor);
nars.emplace(hashPart, narAccessor);
if (cacheDir != "") {
try {
std::ostringstream str;
JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(storePath, "ls"), str.str());
writeFile(makeCacheFile(hashPart, "ls"), str.str());
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(storePath, "nar"), nar);
writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
{
auto path = canonPath(path_);
auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
auto [storePath, restPath] = store->toStorePath(path);
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", storePath);
if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
auto i = nars.find(storePath);
auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
std::string listing;
Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
try {
listing = nix::readFile(makeCacheFile(storePath, "ls"));
listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
auto narAccessor = makeLazyNarAccessor(listing,
[cacheFile](uint64_t offset, uint64_t length) {
@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return buf;
});
nars.emplace(storePath, narAccessor);
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
*sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s);
nars.emplace(storePath, narAccessor);
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
}
store->narFromPath(store->parseStorePath(storePath), sink);
store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath, *sink.s, narAccessor);
addToCache(storePath.hashPart(), *sink.s, narAccessor);
return {narAccessor, restPath};
}

6
src/libstore/remote-fs-accessor.hh
View file

@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
{
ref<Store> store;
std::map<Path, ref<FSAccessor>> nars;
std::map<std::string, ref<FSAccessor>> nars;
Path cacheDir;
@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
friend class BinaryCacheStore;
Path makeCacheFile(const Path & storePath, const std::string & ext);
Path makeCacheFile(std::string_view hashPart, const std::string & ext);
void addToCache(const Path & storePath, const std::string & nar,
void addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor);
public:

42
src/libstore/remote-store.cc
View file

@ -8,6 +8,7 @@
#include "derivations.hh"
#include "pool.hh"
#include "finally.hh"
#include "logging.hh"
#include <sys/types.h>
#include <sys/stat.h>
@ -38,6 +39,29 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i);
}
std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
{
std::map<string, StorePath> pathMap;
auto rawInput = readStrings<Strings>(from);
if (rawInput.size() % 2)
throw Error("got an odd number of elements from the daemon when trying to read a output path map");
auto curInput = rawInput.begin();
while (curInput != rawInput.end()) {
auto thisKey = *curInput++;
auto thisValue = *curInput++;
pathMap.emplace(thisKey, store.parseStorePath(thisValue));
}
return pathMap;
}
void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
{
out << 2*pathMap.size();
for (auto & i : pathMap) {
out << i.first;
out << store.printStorePath(i.second);
}
}
/* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
@ -197,7 +221,7 @@ void RemoteStore::setOptions(Connection & conn)
overrides.erase(settings.maxSilentTime.name);
overrides.erase(settings.buildCores.name);
overrides.erase(settings.useSubstitutes.name);
overrides.erase(settings.showTrace.name);
overrides.erase(loggerSettings.showTrace.name);
conn.to << overrides.size();
for (auto & i : overrides)
conn.to << i.first << i.second.value;
@ -412,12 +436,24 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
{
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
return Store::queryDerivationOutputs(path);
}
conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from);
}
OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return readOutputPathMap(*this, conn->from);
}
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
{
auto conn(getConnection());
@ -430,7 +466,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
RepairFlag repair, CheckSigsFlag checkSigs)
{
auto conn(getConnection());
@ -462,7 +498,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to << wopAddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
<< info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)

4
src/libstore/remote-store.hh
View file

@ -51,6 +51,7 @@ public:
StorePathSet queryDerivationOutputs(const StorePath & path) override;
OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@ -59,8 +60,7 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,

36
src/libstore/s3-binary-cache-store.cc
View file

@ -7,7 +7,6 @@
#include "globals.hh"
#include "compression.hh"
#include "filetransfer.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated;
void uploadFile(const std::string & path, const std::string & data,
void uploadFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
const std::string & contentEncoding)
{
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile(
stream, bucketName, path, mimeType,
istream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */);
@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
if (contentEncoding != "")
request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
request.SetBody(istream);
auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request));
@ -347,25 +343,31 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
bucketName % path % data.size() % duration);
printInfo("uploaded 's3://%s/%s' in %d ms",
bucketName, path, duration);
stats.putTimeMs += duration;
stats.putBytes += data.size();
stats.put++;
}
void upsertFile(const std::string & path, const std::string & data,
void upsertFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto compress = [&](std::string compression)
{
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(*compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
uploadFile(path, compress(lsCompression), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
uploadFile(path, compress(logCompression), mimeType, logCompression);
else
uploadFile(path, data, mimeType, "");
uploadFile(path, istream, mimeType, "");
}
void getFile(const std::string & path, Sink & sink) override
@ -410,7 +412,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
for (auto object : contents) {
auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
}
marker = res.GetNextMarker();

1
src/libstore/s3-binary-cache-store.hh
View file

@ -19,7 +19,6 @@ public:
struct Stats
{
std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0};

9
src/libstore/sqlite.cc
View file

@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt)
});
}
uint64_t SQLite::getLastInsertedRowId()
{
return sqlite3_last_insert_rowid(db);
}
void SQLiteStmt::create(sqlite3 * db, const string & sql)
{
checkInterrupt();
@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use()
sqlite3_reset(stmt);
}
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull)
SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull)
{
if (notNull) {
if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();

4
src/libstore/sqlite.hh
View file

@ -26,6 +26,8 @@ struct SQLite
void isCache();
void exec(const std::string & stmt);
uint64_t getLastInsertedRowId();
};
/* RAII wrapper to create and destroy SQLite prepared statements. */
@ -54,7 +56,7 @@ struct SQLiteStmt
~Use();
/* Bind the next parameter. */
Use & operator () (const std::string & value, bool notNull = true);
Use & operator () (std::string_view value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null

149
src/libstore/store-api.cc
View file

@ -7,6 +7,7 @@
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
#include "archive.hh"
#include <future>
@ -20,15 +21,15 @@ bool Store::isInStore(const Path & path) const
}
Path Store::toStorePath(const Path & path) const
std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
{
if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos)
return path;
return {parseStorePath(path), ""};
else
return Path(path, 0, slash);
return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
}
@ -41,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
throw NotInStore("path '%1%' is not in the Nix store", path);
throw BadStorePath("path '%1%' is not in the Nix store", path);
return path;
}
StorePath Store::followLinksToStorePath(std::string_view path) const
{
return parseStorePath(toStorePath(followLinksToStore(path)));
return toStorePath(followLinksToStore(path)).first;
}
@ -221,6 +222,93 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
}
/*
The aim of this function is to compute in one pass the correct ValidPathInfo for
the files that we are trying to add to the store. To accomplish that in one
pass, given the different kind of inputs that we can take (normal nar archives,
nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks
and aliases. Also, since the dataflow is obfuscated by this, we include here a
graphviz diagram:
digraph graphname {
node [shape=box]
fileSource -> narSink
narSink [style=dashed]
narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"]
narSink -> narHashSink [style = dashed, label = "else"]
unsualHashTee -> narHashSink
unsualHashTee -> caHashSink
fileSource -> parseSink
parseSink [style=dashed]
parseSink-> fileSink [style = dashed, label = "Flat"]
parseSink -> blank [style = dashed, label = "Recursive"]
fileSink -> caHashSink
}
*/
ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
std::optional<Hash> expectedCAHash)
{
HashSink narHashSink { htSHA256 };
HashSink caHashSink { hashAlgo };
/* Note that fileSink and unusualHashTee must be mutually exclusive, since
they both write to caHashSink. Note that that requisite is currently true
because the former is only used in the flat case. */
RetrieveRegularNARSink fileSink { caHashSink };
TeeSink unusualHashTee { narHashSink, caHashSink };
auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
? static_cast<Sink &>(unusualHashTee)
: narHashSink;
/* Functionally, this means that fileSource will yield the content of
srcPath. The fact that we use scratchpadSink as a temporary buffer here
is an implementation detail. */
auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
dumpPath(srcPath, scratchpadSink);
});
/* tapped provides the same data as fileSource, but we also write all the
information to narSink. */
TeeSource tapped { *fileSource, narSink };
ParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat
? fileSink
: blank;
/* The information that flows from tapped (besides being replicated in
narSink), is now put in parseSink. */
parseDump(parseSink, tapped);
/* We extract the result of the computation from the sink by calling
finish. */
auto [narHash, narSize] = narHashSink.finish();
auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
? narHash
: caHashSink.finish().first;
if (expectedCAHash && expectedCAHash != hash)
throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info(makeFixedOutputPath(method, hash, name));
info.narHash = narHash;
info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) {
dumpPath(srcPath, scratchpadSink);
});
addToStore(info, *source);
}
return info;
}
Store::Store(const Params & params)
: Config(params)
, state({(size_t) pathInfoCacheSize})
@ -242,6 +330,16 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
StorePathSet Store::queryDerivationOutputs(const StorePath & path)
{
auto outputMap = this->queryDerivationOutputMap(path);
StorePathSet outputPaths;
for (auto & i: outputMap) {
outputPaths.emplace(std::move(i.second));
}
return outputPaths;
}
bool Store::isValidPath(const StorePath & storePath)
{
std::string hashPart(storePath.hashPart());
@ -306,6 +404,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
}
static bool goodStorePath(const StorePath & expected, const StorePath & actual)
{
return
expected.hashPart() == actual.hashPart()
&& (expected.name() == Store::MissingName || expected.name() == actual.name());
}
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
@ -333,7 +439,7 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
res.second->path != storePath)
!goodStorePath(storePath, res.second->path))
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
return callback(ref<const ValidPathInfo>(res.second));
@ -345,7 +451,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
{[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@ -358,9 +464,11 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
}
if (!info || info->path != parseStorePath(storePath)) {
auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath);
throw InvalidPath("path '%s' is not valid", storePathS);
}
(*callbackPtr)(ref<const ValidPathInfo>(info));
@ -430,7 +538,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
auto info = queryPathInfo(i);
if (showHash) {
s += info->narHash.to_string(Base16, false) + "\n";
s += info->narHash->to_string(Base16, false) + "\n";
s += (format("%1%\n") % info->narSize).str();
}
@ -462,7 +570,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
auto info = queryPathInfo(storePath);
jsonPath
.attr("narHash", info->narHash.to_string(hashBase, true))
.attr("narHash", info->narHash->to_string(hashBase, true))
.attr("narSize", info->narSize);
{
@ -505,7 +613,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize)
@ -746,7 +854,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32, true) + ";"
+ narHash->to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references));
}
@ -859,12 +967,20 @@ ref<Store> openStore(const std::string & uri_,
throw Error("don't know how to open Nix store '%s'", uri);
}
static bool isNonUriPath(const std::string & spec) {
return
// is not a URL
spec.find("://") == std::string::npos
// Has at least one path separator, and so isn't a single word that
// might be special like "auto"
&& spec.find("/") != std::string::npos;
}
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
{
if (uri == "daemon") {
return tDaemon;
} else if (uri == "local" || hasPrefix(uri, "/")) {
} else if (uri == "local" || isNonUriPath(uri)) {
return tLocal;
} else if (uri == "" || uri == "auto") {
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
@ -888,8 +1004,9 @@ static RegisterStoreImplementation regStore([](
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
case tLocal: {
Store::Params params2 = params;
if (hasPrefix(uri, "/"))
params2["root"] = uri;
if (isNonUriPath(uri)) {
params2["root"] = absPath(uri);
}
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
}
default:

42
src/libstore/store-api.hh
View file

@ -31,7 +31,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
MakeError(BadStorePath, Error);
class FSAccessor;
@ -115,7 +115,8 @@ struct ValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
Hash narHash;
// TODO document this
std::optional<Hash> narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
@ -317,9 +318,9 @@ public:
the Nix store. */
bool isStorePath(std::string_view path) const;
/* Chop off the parts after the top-level store name, e.g.,
/nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
Path toStorePath(const Path & path) const;
/* Split a path like /nix/store/<hash>-<name>/<bla> into
/nix/store/<hash>-<name> and /<bla>. */
std::pair<StorePath, Path> toStorePath(const Path & path) const;
/* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(std::string_view path) const;
@ -384,13 +385,16 @@ public:
SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store
backends, the name part of store paths may be omitted
(i.e. you'll get /nix/store/<hash> rather than
backends, the name part of store paths may be replaced by 'x'
(i.e. you'll get /nix/store/<hash>-x rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */
full store path. FIXME: should return a set of
std::variant<StorePath, HashPart> to get rid of this hack. */
virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); }
constexpr static const char * MissingName = "x";
/* Query information about a valid path. It is permitted to omit
the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@ -418,8 +422,11 @@ public:
virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
/* Query the outputs of the derivation denoted by `path'. */
virtual StorePathSet queryDerivationOutputs(const StorePath & path)
{ unsupported("queryDerivationOutputs"); }
virtual StorePathSet queryDerivationOutputs(const StorePath & path);
/* Query the mapping outputName=>outputPath for the given derivation */
virtual OutputPathMap queryDerivationOutputMap(const StorePath & path)
{ unsupported("queryDerivationOutputMap"); }
/* Query the full store path given the hash part of a valid store
path, or empty if the path doesn't exist. */
@ -436,8 +443,7 @@ public:
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) = 0;
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
@ -447,8 +453,15 @@ public:
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of
memory. */
ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
std::optional<Hash> expectedCAHash = {});
// FIXME: remove?
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{
throw Error("addToStoreFromDump() is not supported by this store");
@ -613,8 +626,7 @@ public:
the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent
access. */
StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs);
StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
struct Stats
{

6
src/libstore/worker-protocol.hh
View file

@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x115
#define PROTOCOL_VERSION 0x116
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -30,7 +30,7 @@ typedef enum {
wopSetOptions = 19,
wopCollectGarbage = 20,
wopQuerySubstitutablePathInfo = 21,
wopQueryDerivationOutputs = 22,
wopQueryDerivationOutputs = 22, // obsolete
wopQueryAllValidPaths = 23,
wopQueryFailedPaths = 24,
wopClearFailedPaths = 25,
@ -49,6 +49,7 @@ typedef enum {
wopNarFromPath = 38,
wopAddToStoreNar = 39,
wopQueryMissing = 40,
wopQueryDerivationOutputMap = 41,
} WorkerOp;
@ -69,5 +70,6 @@ template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
}