mirror of
https://github.com/NixOS/nix.git
synced 2025-11-14 14:32:42 +01:00
Merge remote-tracking branch 'upstream/master' into path-info
This commit is contained in:
John Ericson
commit
0e9e599b00
209 changed files with 10886 additions and 3477 deletions
|
|
@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
|||
DerivationOutputs::const_iterator out = drv.outputs.find("out");
|
||||
if (out == drv.outputs.end())
|
||||
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
|
||||
|
||||
check(
|
||||
makeFixedOutputPath(
|
||||
out->second.hash->method,
|
||||
out->second.hash->hash,
|
||||
drvName),
|
||||
out->second.path, "out");
|
||||
}
|
||||
|
||||
else {
|
||||
Hash h = hashDerivationModulo(*this, drv, true);
|
||||
for (auto & i : drv.outputs)
|
||||
check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
|
||||
check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -586,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
|||
|
||||
state.stmtRegisterValidPath.use()
|
||||
(printStorePath(info.path))
|
||||
(info.narHash.to_string(Base16, true))
|
||||
(info.narHash->to_string(Base16, true))
|
||||
(info.registrationTime == 0 ? time(0) : info.registrationTime)
|
||||
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
|
||||
(info.narSize, info.narSize != 0)
|
||||
|
|
@ -594,7 +587,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
|||
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
|
||||
(renderContentAddress(info.ca), (bool) info.ca)
|
||||
.exec();
|
||||
uint64_t id = sqlite3_last_insert_rowid(state.db);
|
||||
uint64_t id = state.db.getLastInsertedRowId();
|
||||
|
||||
/* If this is a derivation, then store the derivation outputs in
|
||||
the database. This is useful for the garbage collector: it can
|
||||
|
|
@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
|||
state.stmtAddDerivationOutput.use()
|
||||
(id)
|
||||
(i.first)
|
||||
(printStorePath(i.second.path))
|
||||
(printStorePath(i.second.path(*this, drv.name)))
|
||||
.exec();
|
||||
}
|
||||
}
|
||||
|
|
@ -688,7 +681,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
|
|||
{
|
||||
state.stmtUpdatePathInfo.use()
|
||||
(info.narSize, info.narSize != 0)
|
||||
(info.narHash.to_string(Base16, true))
|
||||
(info.narHash->to_string(Base16, true))
|
||||
(info.ultimate ? 1 : 0, info.ultimate)
|
||||
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
|
||||
(renderContentAddress(info.ca), (bool) info.ca)
|
||||
|
|
@ -776,17 +769,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
|
|||
}
|
||||
|
||||
|
||||
StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
|
||||
OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
|
||||
{
|
||||
return retrySQLite<StorePathSet>([&]() {
|
||||
return retrySQLite<OutputPathMap>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
|
||||
(queryValidPathId(*state, path)));
|
||||
|
||||
StorePathSet outputs;
|
||||
OutputPathMap outputs;
|
||||
while (useQueryDerivationOutputs.next())
|
||||
outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
|
||||
outputs.emplace(
|
||||
useQueryDerivationOutputs.getStr(0),
|
||||
parseStorePath(useQueryDerivationOutputs.getStr(1))
|
||||
);
|
||||
|
||||
return outputs;
|
||||
});
|
||||
|
|
@ -845,20 +841,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
|
|||
}
|
||||
|
||||
|
||||
void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
|
||||
SubstitutablePathInfos & infos)
|
||||
void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
|
||||
{
|
||||
if (!settings.useSubstitutes) return;
|
||||
for (auto & sub : getDefaultSubstituters()) {
|
||||
if (sub->storeDir != storeDir) continue;
|
||||
for (auto & path : paths) {
|
||||
if (infos.count(path)) continue;
|
||||
debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
|
||||
auto subPath(path.first);
|
||||
|
||||
// recompute store path so that we can use a different store root
|
||||
if (path.second) {
|
||||
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
|
||||
if (sub->storeDir == storeDir)
|
||||
assert(subPath == path.first);
|
||||
if (subPath != path.first)
|
||||
debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
|
||||
} else if (sub->storeDir != storeDir) continue;
|
||||
|
||||
debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
|
||||
try {
|
||||
auto info = sub->queryPathInfo(path);
|
||||
auto info = sub->queryPathInfo(subPath);
|
||||
|
||||
if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
|
||||
continue;
|
||||
|
||||
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
|
||||
std::shared_ptr<const ValidPathInfo>(info));
|
||||
infos.insert_or_assign(path, SubstitutablePathInfo {
|
||||
infos.insert_or_assign(path.first, SubstitutablePathInfo{
|
||||
info->references,
|
||||
info->hasSelfReference,
|
||||
info->deriver,
|
||||
|
|
@ -901,7 +909,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
|||
StorePathSet paths;
|
||||
|
||||
for (auto & i : infos) {
|
||||
assert(i.narHash.type == htSHA256);
|
||||
assert(i.narHash && i.narHash->type == htSHA256);
|
||||
if (isValidPath_(*state, i.path))
|
||||
updatePathInfo(*state, i);
|
||||
else
|
||||
|
|
@ -963,7 +971,7 @@ const PublicKeys & LocalStore::getPublicKeys()
|
|||
|
||||
|
||||
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
|
||||
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
{
|
||||
if (!info.narHash)
|
||||
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
|
||||
|
|
@ -977,7 +985,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
|
||||
PathLocks outputLock;
|
||||
|
||||
Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
|
||||
auto realPath = Store::toRealPath(info.path);
|
||||
|
||||
/* Lock the output path. But don't lock if we're being called
|
||||
from a build hook (whose parent process already acquired a
|
||||
|
|
@ -1013,7 +1021,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
|
||||
if (hashResult.first != info.narHash)
|
||||
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
|
||||
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
|
||||
printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true));
|
||||
|
||||
if (hashResult.second != info.narSize)
|
||||
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
|
||||
|
|
@ -1033,12 +1041,78 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
|
||||
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
readFile(srcPath, sink);
|
||||
});
|
||||
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
|
||||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
|
||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
||||
{
|
||||
Hash h = hashString(hashAlgo, dump);
|
||||
/* For computing the store path. */
|
||||
auto hashSink = std::make_unique<HashSink>(hashAlgo);
|
||||
TeeSource source { source0, *hashSink };
|
||||
|
||||
auto dstPath = makeFixedOutputPath(method, h, name);
|
||||
/* Read the source path into memory, but only if it's up to
|
||||
narBufferSize bytes. If it's larger, write it to a temporary
|
||||
location in the Nix store. If the subsequently computed
|
||||
destination store path is already valid, we just delete the
|
||||
temporary path. Otherwise, we move it to the destination store
|
||||
path. */
|
||||
bool inMemory = false;
|
||||
|
||||
std::string dump;
|
||||
|
||||
/* Fill out buffer, and decide whether we are working strictly in
|
||||
memory based on whether we break out because the buffer is full
|
||||
or the original source is empty */
|
||||
while (dump.size() < settings.narBufferSize) {
|
||||
auto oldSize = dump.size();
|
||||
constexpr size_t chunkSize = 65536;
|
||||
auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
|
||||
dump.resize(oldSize + want);
|
||||
auto got = 0;
|
||||
try {
|
||||
got = source.read((uint8_t *) dump.data() + oldSize, want);
|
||||
} catch (EndOfFile &) {
|
||||
inMemory = true;
|
||||
break;
|
||||
}
|
||||
dump.resize(oldSize + got);
|
||||
}
|
||||
|
||||
std::unique_ptr<AutoDelete> delTempDir;
|
||||
Path tempPath;
|
||||
|
||||
if (!inMemory) {
|
||||
/* Drain what we pulled so far, and then keep on pulling */
|
||||
StringSource dumpSource { dump };
|
||||
ChainSource bothSource { dumpSource, source };
|
||||
|
||||
auto tempDir = createTempDir(realStoreDir, "add");
|
||||
delTempDir = std::make_unique<AutoDelete>(tempDir);
|
||||
tempPath = tempDir + "/x";
|
||||
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
restorePath(tempPath, bothSource);
|
||||
else
|
||||
writeFile(tempPath, bothSource);
|
||||
|
||||
dump.clear();
|
||||
}
|
||||
|
||||
auto [hash, size] = hashSink->finish();
|
||||
|
||||
auto dstPath = makeFixedOutputPath(method, hash, name);
|
||||
|
||||
addTempRoot(dstPath);
|
||||
|
||||
|
|
@ -1047,8 +1121,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
/* The first check above is an optimisation to prevent
|
||||
unnecessary lock acquisition. */
|
||||
|
||||
Path realPath = realStoreDir + "/";
|
||||
realPath += dstPath.to_string();
|
||||
auto realPath = Store::toRealPath(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
|
|
@ -1058,31 +1131,36 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
|
||||
autoGC();
|
||||
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
StringSource source(dump);
|
||||
restorePath(realPath, source);
|
||||
} else
|
||||
writeFile(realPath, dump);
|
||||
if (inMemory) {
|
||||
StringSource dumpSource { dump };
|
||||
/* Restore from the NAR in memory. */
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
restorePath(realPath, dumpSource);
|
||||
else
|
||||
writeFile(realPath, dumpSource);
|
||||
} else {
|
||||
/* Move the temporary path we restored above. */
|
||||
if (rename(tempPath.c_str(), realPath.c_str()))
|
||||
throw Error("renaming '%s' to '%s'", tempPath, realPath);
|
||||
}
|
||||
|
||||
canonicalisePathMetaData(realPath, -1);
|
||||
/* For computing the nar hash. In recursive SHA-256 mode, this
|
||||
is the same as the store hash, so no need to do it again. */
|
||||
auto narHash = std::pair { hash, size };
|
||||
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
|
||||
HashSink narSink { htSHA256 };
|
||||
dumpPath(realPath, narSink);
|
||||
narHash = narSink.finish();
|
||||
}
|
||||
|
||||
/* Register the SHA-256 hash of the NAR serialisation of
|
||||
the path in the database. We may just have computed it
|
||||
above (if called with recursive == true and hashAlgo ==
|
||||
sha256); otherwise, compute it here. */
|
||||
HashResult hash;
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
|
||||
hash.second = dump.size();
|
||||
} else
|
||||
hash = hashPath(htSHA256, realPath);
|
||||
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
|
||||
|
||||
optimisePath(realPath); // FIXME: combine with hashPath()
|
||||
optimisePath(realPath);
|
||||
|
||||
ValidPathInfo info(dstPath);
|
||||
info.narHash = hash.first;
|
||||
info.narSize = hash.second;
|
||||
info.ca = FixedOutputHash { .method = method, .hash = h };
|
||||
info.narHash = narHash.first;
|
||||
info.narSize = narHash.second;
|
||||
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
||||
registerValidPath(info);
|
||||
}
|
||||
|
||||
|
|
@ -1093,24 +1171,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
|||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
|
||||
/* Read the whole path into memory. This is not a very scalable
|
||||
method for very large paths, but `copyPath' is mainly used for
|
||||
small files. */
|
||||
StringSink sink;
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
sink.s = make_ref<std::string>(readFile(srcPath));
|
||||
|
||||
return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
|
||||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair)
|
||||
{
|
||||
|
|
@ -1121,8 +1181,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
|
|||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
Path realPath = realStoreDir + "/";
|
||||
realPath += dstPath.to_string();
|
||||
auto realPath = Store::toRealPath(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
|
|
@ -1258,9 +1317,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
|
|||
|
||||
std::unique_ptr<AbstractHashSink> hashSink;
|
||||
if (!info->ca || !info->hasSelfReference)
|
||||
hashSink = std::make_unique<HashSink>(*info->narHash.type);
|
||||
hashSink = std::make_unique<HashSink>(info->narHash->type);
|
||||
else
|
||||
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
|
||||
hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
|
||||
|
||||
dumpPath(Store::toRealPath(i), *hashSink);
|
||||
auto current = hashSink->finish();
|
||||
|
|
@ -1269,7 +1328,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
|
|||
logError({
|
||||
.name = "Invalid hash - path modified",
|
||||
.hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
|
||||
printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
|
||||
printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true))
|
||||
});
|
||||
if (repair) repairPath(i); else errors = true;
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue