nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-20 17:29:36 +01:00
Code Activity

Merge remote-tracking branch 'origin/master' into flakes

Browse source
This commit is contained in:
Eelco Dolstra 2020-06-26 08:46:46 +02:00
commit adf2fbbdc2
66 changed files with 751 additions and 359 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/libstore/binary-cache-store.cc
View file

@ -388,8 +388,6 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
narInfo->sigs.insert(sigs.begin(), sigs.end());
auto narInfoFile = narInfoFileFor(narInfo->path);
writeNarInfo(narInfo);
}

41
src/libstore/build.cc
View file

@ -86,7 +86,7 @@ struct HookInstance;
/* A pointer to a goal. */
class Goal;
struct Goal;
class DerivationGoal;
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
@ -1195,6 +1195,12 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (parsedDrv->contentAddressed()) {
settings.requireExperimentalFeature("ca-derivations");
throw Error("ca-derivations isn't implemented yet");
}
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
@ -3708,14 +3714,11 @@ void DerivationGoal::registerOutputs()
/* Check that fixed-output derivations produced the right
outputs (i.e., the content hash should match the specified
hash). */
std::string ca;
std::optional<ContentAddress> ca;
if (fixedOutput) {
FileIngestionMethod outputHashMode; Hash h;
i.second.parseHashInfo(outputHashMode, h);
if (outputHashMode == FileIngestionMethod::Flat) {
if (i.second.hash->method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@ -3726,20 +3729,22 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = outputHashMode == FileIngestionMethod::Recursive
? hashPath(h.type, actualPath).first
: hashFile(h.type, actualPath);
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
? hashPath(*i.second.hash->hash.type, actualPath).first
: hashFile(*i.second.hash->hash.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
if (h != h2) {
if (i.second.hash->hash != h2) {
/* Throw an error after registering the path as
valid. */
worker.hashMismatch = true;
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true)));
worker.store.printStorePath(dest),
i.second.hash->hash.to_string(SRI, true),
h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest);
@ -3759,7 +3764,10 @@ void DerivationGoal::registerOutputs()
else
assert(worker.store.parseStorePath(path) == dest);
ca = makeFixedOutputCA(outputHashMode, h2);
ca = FixedOutputHash {
.method = i.second.hash->method,
.hash = h2,
};
}
/* Get rid of all weird permissions. This also checks that
@ -3832,7 +3840,10 @@ void DerivationGoal::registerOutputs()
info.ca = ca;
worker.store.signPathInfo(info);
if (!info.references.empty()) info.ca.clear();
if (!info.references.empty()) {
// FIXME don't we have an experimental feature for fixed output with references?
info.ca = {};
}
infos.emplace(i.first, std::move(info));
}
@ -4992,7 +5003,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}

4
src/libstore/builtins/fetchurl.cc
View file

@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto ht = parseHashType(getAttr("outputHashAlgo"));
auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());

85
src/libstore/content-address.cc Normal file
View file

@ -0,0 +1,85 @@
#include "content-address.hh"
namespace nix {
std::string FixedOutputHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
}
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
switch (m) {
case FileIngestionMethod::Flat:
return "";
case FileIngestionMethod::Recursive:
return "r:";
default:
throw Error("impossible, caught both cases");
}
}
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
{
return "fixed:"
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base32, true);
}
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
std::string renderContentAddress(ContentAddress ca) {
return std::visit(overloaded {
[](TextHash th) {
return "text:" + th.hash.to_string(Base32, true);
},
[](FixedOutputHash fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash);
}
}, ca);
}
ContentAddress parseContentAddress(std::string_view rawCa) {
auto prefixSeparator = rawCa.find(':');
if (prefixSeparator != string::npos) {
auto prefix = string(rawCa, 0, prefixSeparator);
if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
Hash hash = Hash(string(hashTypeAndHash));
if (*hash.type != htSHA256) {
throw Error("parseContentAddress: the text hash should have type SHA256");
}
return TextHash { hash };
} else if (prefix == "fixed") {
// This has to be an inverse of makeFixedOutputCA
auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
if (methodAndHash.substr(0,2) == "r:") {
std::string_view hashRaw = methodAndHash.substr(2,string::npos);
return FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = Hash(string(hashRaw)),
};
} else {
std::string_view hashRaw = methodAndHash;
return FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash(string(hashRaw)),
};
}
} else {
throw Error("parseContentAddress: format not recognized; has to be text or fixed");
}
} else {
throw Error("Not a content address because it lacks an appropriate prefix");
}
};
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
};
std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : "";
}
}

56
src/libstore/content-address.hh Normal file
View file

@ -0,0 +1,56 @@
#pragma once
#include <variant>
#include "hash.hh"
namespace nix {
enum struct FileIngestionMethod : uint8_t {
Flat = false,
Recursive = true
};
struct TextHash {
Hash hash;
};
/// Pair of a hash, and how the file system was ingested
struct FixedOutputHash {
FileIngestionMethod method;
Hash hash;
std::string printMethodAlgo() const;
};
/*
We've accumulated several types of content-addressed paths over the years;
fixed-output derivations support multiple hash algorithms and serialisation
methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
* ‘text:sha256:<sha256 hash of file contents>’: For paths
computed by makeTextPath() / addTextToStore().
* ‘fixed:<r?>:<ht>:<h>’: For paths computed by
makeFixedOutputPath() / addToStore().
*/
typedef std::variant<
TextHash, // for paths computed by makeTextPath() / addTextToStore
FixedOutputHash // for path computed by makeFixedOutputPath
> ContentAddress;
/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
std::string renderContentAddress(ContentAddress ca);
std::string renderContentAddress(std::optional<ContentAddress> ca);
ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
}

5
src/libstore/daemon.cc
View file

@ -652,7 +652,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
<< info->ca;
<< renderContentAddress(info->ca);
}
} else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@ -710,7 +710,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.references = readStorePaths<StorePathSet>(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
from >> info.ca >> repair >> dontCheckSigs;
info.ca = parseContentAddressOpt(readString(from));
from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs)
dontCheckSigs = false;
if (!trusted)

116
src/libstore/derivations.cc
View file

@ -8,25 +8,6 @@
namespace nix {
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{
recursive = FileIngestionMethod::Flat;
string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") {
recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2);
}
HashType hashType = parseHashType(algo);
if (hashType == htUnknown)
throw Error("unknown hash algorithm '%s'", algo);
hash = Hash(this->hash, hashType);
}
const StorePath & BasicDerivation::findOutput(const string & id) const
{
auto i = outputs.find(id);
@ -120,6 +101,34 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
static Derivation parseDerivation(const Store & store, const string & s)
{
Derivation drv;
@ -129,15 +138,8 @@ static Derivation parseDerivation(const Store & store, const string & s)
/* Parse the list of outputs. */
while (!endOfList(str)) {
expect(str, "("); std::string id = parseString(str);
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); auto hash = parseString(str);
expect(str, ")");
drv.outputs.emplace(id, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
auto output = parseDerivationOutput(store, str);
drv.outputs.emplace(std::move(id), std::move(output));
}
/* Parse the list of input derivations. */
@ -263,8 +265,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
s += ','; printUnquotedString(s, i.second.hashAlgo);
s += ','; printUnquotedString(s, i.second.hash);
s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
s += ','; printUnquotedString(s,
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
s += ')';
}
@ -320,7 +323,7 @@ bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
outputs.begin()->second.hash != "";
outputs.begin()->second.hash;
}
@ -353,8 +356,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ i->second.hash->printMethodAlgo() + ":"
+ i->second.hash->hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path));
}
@ -397,6 +400,31 @@ StorePathSet BasicDerivation::outputPaths() const
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
auto hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
StringSet BasicDerivation::outputNames() const
{
@ -413,14 +441,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
auto output = readDerivationOutput(in, store);
drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@ -441,8 +463,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
for (auto & i : drv.outputs)
out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
for (auto & i : drv.outputs) {
out << i.first
<< store.printStorePath(i.second.path);
if (i.second.hash) {
out << i.second.hash->printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false);
} else {
out << "" << "";
}
}
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();

8
src/libstore/derivations.hh
View file

@ -1,8 +1,9 @@
#pragma once
#include "path.hh"
#include "types.hh"
#include "hash.hh"
#include "store-api.hh"
#include "content-address.hh"
#include <map>
@ -15,9 +16,7 @@ namespace nix {
struct DerivationOutput
{
StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -70,6 +69,7 @@ struct Derivation : BasicDerivation
class Store;
enum RepairFlag : bool { NoRepair = false, Repair = true };
/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store,

2
src/libstore/export-import.cc
View file

@ -55,7 +55,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));

6
src/libstore/filetransfer.cc
View file

@ -73,6 +73,7 @@ struct curlFileTransfer : public FileTransfer
curl_off_t writtenToSink = 0;
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
/* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus()
{
@ -99,7 +100,7 @@ struct curlFileTransfer : public FileTransfer
/* Only write data to the sink if this is a
successful response. */
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
if (successfulStatuses.count(httpStatus)) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
}
@ -356,8 +357,7 @@ struct curlFileTransfer : public FileTransfer
if (writeException)
failEx(writeException);
else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
{
result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize);

4
src/libstore/legacy-ssh-store.cc
View file

@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? Hash() : Hash(s);
conn->from >> info->ca;
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}
@ -146,7 +146,7 @@ struct LegacySSHStore : public Store
<< info.narSize
<< info.ultimate
<< info.sigs
<< info.ca;
<< renderContentAddress(info.ca);
try {
copyNAR(source, conn->to);
} catch (...) {

36
src/libstore/local-store.cc
View file

@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
FileIngestionMethod method; Hash h;
out->second.parseHashInfo(method, h);
check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
check(
makeFixedOutputPath(
out->second.hash->method,
out->second.hash->hash,
drvName),
out->second.path, "out");
}
else {
@ -578,7 +580,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
if (info.ca != "" && !info.isContentAddressed(*this))
if (info.ca.has_value() && !info.isContentAddressed(*this))
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path));
@ -590,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state,
(info.narSize, info.narSize != 0)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(info.ca, !info.ca.empty())
(renderContentAddress(info.ca), (bool) info.ca)
.exec();
uint64_t id = state.db.getLastInsertedRowId();
@ -664,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
if (s) info->ca = s;
if (s) info->ca = parseContentAddressOpt(s);
/* Get the references. */
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@ -687,7 +689,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
(info.narHash.to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(info.ca, !info.ca.empty())
(renderContentAddress(info.ca), (bool) info.ca)
(printStorePath(info.path))
.exec();
}
@ -983,15 +985,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
deletePath(realPath);
if (info.ca != "" &&
!((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
|| info.references.empty()))
// text hashing has long been allowed to have non-self-references because it is used for drv files.
bool refersToSelf = info.references.count(info.path) > 0;
if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
settings.requireExperimentalFeature("ca-references");
/* While restoring the path from the NAR, compute the hash
of the NAR. */
std::unique_ptr<AbstractHashSink> hashSink;
if (info.ca == "" || !info.references.count(info.path))
if (!info.ca.has_value() || !info.references.count(info.path))
hashSink = std::make_unique<HashSink>(htSHA256);
else
hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
@ -1077,7 +1079,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
ValidPathInfo info(dstPath);
info.narHash = hash.first;
info.narSize = hash.second;
info.ca = makeFixedOutputCA(method, h);
info.ca = FixedOutputHash { .method = method, .hash = h };
registerValidPath(info);
}
@ -1141,7 +1143,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
info.narHash = narHash;
info.narSize = sink.s->size();
info.references = references;
info.ca = "text:" + hash.to_string(Base32, true);
info.ca = TextHash { .hash = hash };
registerValidPath(info);
}
@ -1252,10 +1254,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "" || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(info->narHash.type);
if (!info->ca || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(*info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();

4
src/libstore/nar-info-disk-cache.cc
View file

@ -203,7 +203,7 @@ public:
narInfo->deriver = StorePath(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
narInfo->ca = queryNAR.getStr(11);
narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11));
return {oValid, narInfo};
});
@ -237,7 +237,7 @@ public:
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
(concatStringsSep(" ", info->sigs))
(info->ca)
(renderContentAddress(info->ca))
(time(0)).exec();
} else {

9
src/libstore/nar-info.cc
View file

@ -67,8 +67,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
if (!ca.empty()) corrupt();
ca = value;
if (ca) corrupt();
// FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value);
}
pos = eol + 1;
@ -104,8 +105,8 @@ std::string NarInfo::to_string(const Store & store) const
for (auto sig : sigs)
res += "Sig: " + sig + "\n";
if (!ca.empty())
res += "CA: " + ca + "\n";
if (ca)
res += "CA: " + renderContentAddress(*ca) + "\n";
return res;
}

5
src/libstore/parsed-derivations.cc
View file

@ -117,4 +117,9 @@ bool ParsedDerivation::substitutesAllowed() const
return getBoolAttr("allowSubstitutes", true);
}
bool ParsedDerivation::contentAddressed() const
{
return getBoolAttr("__contentAddressed", false);
}
}

4
src/libstore/parsed-derivations.hh
View file

@ -1,4 +1,4 @@
#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json_fwd.hpp>
@ -34,6 +34,8 @@ public:
bool willBuildLocally() const;
bool substitutesAllowed() const;
bool contentAddressed() const;
};
}

6
src/libstore/path.hh
View file

@ -1,5 +1,6 @@
#pragma once
#include "content-address.hh"
#include "types.hh"
namespace nix {
@ -65,11 +66,6 @@ typedef std::vector<StorePath> StorePaths;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
enum struct FileIngestionMethod : uint8_t {
Flat = false,
Recursive = true
};
struct StorePathWithOutputs
{
StorePath path;

6
src/libstore/remote-store.cc
View file

@ -228,7 +228,7 @@ struct ConnectionHandle
~ConnectionHandle()
{
if (!daemonException && std::uncaught_exception()) {
if (!daemonException && std::uncaught_exceptions()) {
handle.markBad();
debug("closing daemon connection because of an exception");
}
@ -381,7 +381,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
conn->from >> info->ultimate;
info->sigs = readStrings<StringSet>(conn->from);
conn->from >> info->ca;
info->ca = parseContentAddressOpt(readString(conn->from));
}
}
callback(std::move(info));
@ -465,7 +465,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
<< info.narHash.to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << info.ca
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
if (!tunnel) copyNAR(source, conn->to);

89
src/libstore/store-api.cc
View file

@ -61,7 +61,7 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
/* Store paths have the following form:
<store>/<h>-<name>
<realized-path> = <store>/<h>-<name>
where
@ -85,11 +85,14 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
<type> = one of:
"text:<r1>:<r2>:...<rN>"
for plain text files written to the store using
addTextToStore(); <r1> ... <rN> are the references of the
path.
"source"
addTextToStore(); <r1> ... <rN> are the store paths referenced
by this path, in the form described by <realized-path>
"source:<r1>:<r2>:...:<rN>:self"
for paths copied to the store using addToStore() when recursive
= true and hashAlgo = "sha256"
= true and hashAlgo = "sha256". Just like in the text case, we
can have the store paths referenced by the path.
Additionally, we can have an optional :self label to denote self
reference.
"output:<id>"
for either the outputs created by derivations, OR paths copied
to the store using addToStore() with recursive != true or
@ -117,6 +120,12 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
the contents of the path (or expected contents of the
path for fixed-output derivations)
Note that since an output derivation has always type output, while
something added by addToStore can have type output or source depending
on the hash, this means that the same input can be hashed differently
if added to the store via addToStore or via a derivation, in the sha256
recursive case.
It would have been nicer to handle fixed-output derivations under
"source", e.g. have something like "source:<rec><algo>", but we're
stuck with this for now...
@ -164,20 +173,20 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(
FileIngestionMethod recursive,
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{
if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
+ (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base16, true) + ":"),
name);
}
@ -462,8 +471,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
jsonRefs.elem(printStorePath(ref));
}
if (info->ca != "")
jsonPath.attr("ca", info->ca);
if (info->ca)
jsonPath.attr("ca", renderContentAddress(info->ca));
std::pair<uint64_t, uint64_t> closureSizes;
@ -748,41 +757,35 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto warn = [&]() {
logWarning(
ErrorInfo{
.name = "Path not content-addressed",
.hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
});
};
if (! ca) return false;
if (hasPrefix(ca, "text:")) {
Hash hash(ca.substr(5));
if (store.makeTextPath(path.name(), hash, references) == path)
return true;
else
warn();
}
else if (hasPrefix(ca, "fixed:")) {
FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6));
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
hasSelfReference = true;
refs.erase(path);
auto caPath = std::visit(overloaded {
[&](TextHash th) {
return store.makeTextPath(path.name(), th.hash, references);
},
[&](FixedOutputHash fsh) {
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
hasSelfReference = true;
refs.erase(path);
}
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference);
}
if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path)
return true;
else
warn();
}
}, *ca);
return false;
bool res = caPath == path;
if (!res)
printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
return res;
}
@ -813,14 +816,6 @@ Strings ValidPathInfo::shortRefs() const
}
std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
{
return "fixed:"
+ (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+ hash.to_string(Base32, true);
}
}

36
src/libstore/store-api.hh
View file

@ -2,12 +2,14 @@
#include "path.hh"
#include "hash.hh"
#include "content-address.hh"
#include "serialise.hh"
#include "crypto.hh"
#include "lru-cache.hh"
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
#include "derivations.hh"
#include <atomic>
#include <limits>
@ -17,6 +19,7 @@
#include <memory>
#include <string>
#include <chrono>
#include <variant>
namespace nix {
@ -31,15 +34,12 @@ MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
struct BasicDerivation;
struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store;
class JSONPlaceholder;
enum RepairFlag : bool { NoRepair = false, Repair = true };
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
@ -111,7 +111,6 @@ struct SubstitutablePathInfo
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
StorePath path;
@ -140,21 +139,11 @@ struct ValidPathInfo
that a particular output path was produced by a derivation; the
path then implies the contents.)
Ideally, the content-addressability assertion would just be a
Boolean, and the store path would be computed from
the name component, ‘narHash’ and ‘references’. However,
1) we've accumulated several types of content-addressed paths
over the years; and 2) fixed-output derivations support
multiple hash algorithms and serialisation methods (flat file
vs NAR). Thus, ‘ca’ has one of the following forms:
* ‘text:sha256:<sha256 hash of file contents>’: For paths
computed by makeTextPath() / addTextToStore().
* ‘fixed:<r?>:<ht>:<h>’: For paths computed by
makeFixedOutputPath() / addToStore().
Ideally, the content-addressability assertion would just be a Boolean,
and the store path would be computed from the name component, ‘narHash’
and ‘references’. However, we support many types of content addresses.
*/
std::string ca;
std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
@ -189,9 +178,10 @@ struct ValidPathInfo
Strings shortRefs() const;
ValidPathInfo(const StorePath & path) : path(path) { }
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { }
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
@ -838,12 +828,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
bool hashGiven = false);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);