nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-20 17:29:36 +01:00
Code Activity

Merge remote-tracking branch 'origin/master' into readd-hashed-mirrors

Browse source
This commit is contained in:
Matthew Bauer 2020-08-06 18:58:59 -05:00
commit 8abc577cc2
50 changed files with 971 additions and 464 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/libstore/binary-cache-store.cc
View file

@ -169,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
fileSink.flush();
}
auto now2 = std::chrono::steady_clock::now();
@ -282,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
"application/x-nix-nar");
} else
stats.narWriteAverted++;

81
src/libstore/build.cc
View file

@ -806,8 +806,8 @@ private:
/* RAII object to delete the chroot directory. */
std::shared_ptr<AutoDelete> autoDelChroot;
/* Whether this is a fixed-output derivation. */
bool fixedOutput;
/* The sort of derivation we are building. */
DerivationType derivationType;
/* Whether to run the build in a private network namespace. */
bool privateNetwork = false;
@ -1195,9 +1195,9 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (parsedDrv->contentAddressed()) {
if (drv->type() == DerivationType::CAFloating) {
settings.requireExperimentalFeature("ca-derivations");
throw Error("ca-derivations isn't implemented yet");
throw UnimplementedError("ca-derivations isn't implemented yet");
}
@ -1392,12 +1392,12 @@ void DerivationGoal::inputsRealised()
debug("added input paths %s", worker.store.showPaths(inputPaths));
/* Is this a fixed-output derivation? */
fixedOutput = drv->isFixedOutput();
/* What type of derivation are we building? */
derivationType = drv->type();
/* Don't repeat fixed-output derivations since they're already
verified by their output hash.*/
nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1;
nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
/* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a
@ -1783,7 +1783,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
fixedOutput || diskFull ? BuildResult::TransientFailure :
derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@ -1996,7 +1996,7 @@ void DerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
useChroot = !fixedOutput && !noChroot;
useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
}
if (worker.store.storeDir != worker.store.realStoreDir) {
@ -2165,7 +2165,7 @@ void DerivationGoal::startBuilder()
"nogroup:x:65534:\n") % sandboxGid).str());
/* Create /etc/hosts with localhost entry. */
if (!fixedOutput)
if (!(derivationIsImpure(derivationType)))
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@ -2373,7 +2373,7 @@ void DerivationGoal::startBuilder()
us.
*/
if (!fixedOutput)
if (!(derivationIsImpure(derivationType)))
privateNetwork = true;
userNamespaceSync.create();
@ -2574,7 +2574,7 @@ void DerivationGoal::initEnv()
derivation, tell the builder, so that for instance `fetchurl'
can skip checking the output. On older Nixes, this environment
variable won't be set, so `fetchurl' will do the check. */
if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
/* *Only* if this is a fixed-output derivation, propagate the
values of the environment variables specified in the
@ -2585,7 +2585,7 @@ void DerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
if (fixedOutput) {
if (derivationIsImpure(derivationType)) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@ -3195,7 +3195,7 @@ void DerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
if (fixedOutput) {
if (derivationIsImpure(derivationType)) {
ss.push_back("/etc/resolv.conf");
// Only use nss functions to resolve hosts and
@ -3436,7 +3436,7 @@ void DerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
if (fixedOutput)
if (derivationIsImpure(derivationType))
sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Our rwx outputs */
@ -3721,9 +3721,22 @@ void DerivationGoal::registerOutputs()
hash). */
std::optional<ContentAddress> ca;
if (fixedOutput) {
FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash;
if (! std::holds_alternative<DerivationOutputInputAddressed>(i.second.output)) {
DerivationOutputCAFloating outputHash;
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
assert(false); // Enclosing `if` handles this case in other branch
},
[&](DerivationOutputCAFixed dof) {
outputHash = DerivationOutputCAFloating {
.method = dof.hash.method,
.hashType = dof.hash.hash.type,
};
},
[&](DerivationOutputCAFloating dof) {
outputHash = dof;
},
}, i.second.output);
if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
@ -3737,12 +3750,17 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = outputHash.method == FileIngestionMethod::Recursive
? hashPath(outputHash.hash.type, actualPath).first
: hashFile(outputHash.hash.type, actualPath);
? hashPath(outputHash.hashType, actualPath).first
: hashFile(outputHash.hashType, actualPath);
auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
if (outputHash.hash != h2) {
// true if either floating CA, or incorrect fixed hash.
bool needsMove = true;
if (auto p = std::get_if<DerivationOutputCAFixed>(& i.second.output)) {
Hash & h = p->hash.hash;
if (h != h2) {
/* Throw an error after registering the path as
valid. */
@ -3750,9 +3768,15 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest),
outputHash.hash.to_string(SRI, true),
h.to_string(SRI, true),
h2.to_string(SRI, true)));
} else {
// matched the fixed hash, so no move needed.
needsMove = false;
}
}
if (needsMove) {
Path actualDest = worker.store.Store::toRealPath(dest);
if (worker.store.isValidPath(dest))
@ -4852,8 +4876,17 @@ void Worker::run(const Goals & _topGoals)
waitForInput();
else {
if (awake.empty() && 0 == settings.maxBuildJobs)
throw Error("unable to start any build; either increase '--max-jobs' "
"or enable remote builds");
{
if (getMachines().empty())
throw Error("unable to start any build; either increase '--max-jobs' "
"or enable remote builds."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
else
throw Error("unable to start any build; remote machines may not have "
"all required system features."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
}
assert(!awake.empty());
}
}

76
src/libstore/content-address.cc
View file

@ -1,4 +1,6 @@
#include "args.hh"
#include "content-address.hh"
#include "split.hh"
namespace nix {
@ -24,10 +26,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+ hash.to_string(Base32, true);
}
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
std::string renderContentAddress(ContentAddress ca) {
return std::visit(overloaded {
[](TextHash th) {
@ -40,38 +38,46 @@ std::string renderContentAddress(ContentAddress ca) {
}
ContentAddress parseContentAddress(std::string_view rawCa) {
auto prefixSeparator = rawCa.find(':');
if (prefixSeparator != string::npos) {
auto prefix = string(rawCa, 0, prefixSeparator);
if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
Hash hash = Hash(string(hashTypeAndHash));
if (hash.type != htSHA256) {
throw Error("parseContentAddress: the text hash should have type SHA256");
}
return TextHash { hash };
} else if (prefix == "fixed") {
// This has to be an inverse of makeFixedOutputCA
auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
if (methodAndHash.substr(0,2) == "r:") {
std::string_view hashRaw = methodAndHash.substr(2,string::npos);
return FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = Hash(string(hashRaw)),
};
} else {
std::string_view hashRaw = methodAndHash;
return FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash(string(hashRaw)),
};
}
} else {
throw Error("parseContentAddress: format not recognized; has to be text or fixed");
}
} else {
throw Error("Not a content address because it lacks an appropriate prefix");
auto rest = rawCa;
std::string_view prefix;
{
auto optPrefix = splitPrefixTo(rest, ':');
if (!optPrefix)
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", rawCa);
prefix = *optPrefix;
}
auto parseHashType_ = [&](){
auto hashTypeRaw = splitPrefixTo(rest, ':');
if (!hashTypeRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", rawCa);
HashType hashType = parseHashType(*hashTypeRaw);
return std::move(hashType);
};
// Switch on prefix
if (prefix == "text") {
// No parsing of the method, "text" only support flat.
HashType hashType = parseHashType_();
if (hashType != htSHA256)
throw Error("text content address hash should use %s, but instead uses %s",
printHashType(htSHA256), printHashType(hashType));
return TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
};
} else if (prefix == "fixed") {
// Parse method
auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_();
return FixedOutputHash {
.method = method,
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
};
} else
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
};
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {

2
src/libstore/daemon.cc
View file

@ -698,7 +698,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto deriver = readString(from);
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.narHash = Hash(readString(from), htSHA256);
info.narHash = Hash::parseAny(readString(from), htSHA256);
info.references = readStorePaths<StorePathSet>(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);

289
src/libstore/derivations.cc
View file

@ -7,23 +7,54 @@
namespace nix {
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
{
return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) {
return doi.path;
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
return { doi.path };
},
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return {
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
};
},
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
return std::nullopt;
},
[&](DerivationOutputFixed dof) {
return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
}
}, output);
}
bool derivationIsCA(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return true;
};
// Since enums can have non-variant values, but making a `default:` would
// disable exhaustiveness warnings.
assert(false);
}
bool derivationIsFixed(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
};
assert(false);
}
bool derivationIsImpure(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
};
assert(false);
}
bool BasicDerivation::isBuiltin() const
{
return string(builder, 0, 8) == "builtin:";
@ -123,14 +154,22 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
}
const HashType hashType = parseHashType(hashAlgo);
return DerivationOutput {
.output = DerivationOutputFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
},
}
};
return hash != ""
? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
@ -278,13 +317,20 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
} else {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
}, i.second.output);
s += ')';
}
@ -336,60 +382,134 @@ bool isDerivation(const string & fileName)
}
bool BasicDerivation::isFixedOutput() const
DerivationType BasicDerivation::type() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output);
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
[&](DerivationOutputInputAddressed _) {
inputAddressedOutputs.insert(i.first);
},
[&](DerivationOutputCAFixed _) {
fixedCAOutputs.insert(i.first);
},
[&](DerivationOutputCAFloating dof) {
floatingCAOutputs.insert(i.first);
if (!floatingHashType) {
floatingHashType = dof.hashType;
} else {
if (*floatingHashType != dof.hashType)
throw Error("All floating outputs must use the same hash type");
}
},
}, i.second.output);
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
return DerivationType::InputAddressed;
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
return DerivationType::CAFixed;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
return DerivationType::CAFloating;
} else {
throw Error("Can't mix derivation output types");
}
}
DrvHashes drvHashes;
/* pathDerivationModulo and hashDerivationModulo are mutually recursive
*/
/* Returns the hash of a derivation modulo fixed-output
subderivations. A fixed-output derivation is a derivation with one
output (`out') for which an expected hash and hash algorithm are
specified (using the `outputHash' and `outputHashAlgo'
attributes). We don't want changes to such derivations to
propagate upwards through the dependency graph, changing output
paths everywhere.
/* Look up the derivation by value and memoize the
`hashDerivationModulo` call.
*/
static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
{
auto h = drvHashes.find(drvPath);
if (h == drvHashes.end()) {
assert(store.isValidPath(drvPath));
// Cache it
h = drvHashes.insert_or_assign(
drvPath,
hashDerivationModulo(
store,
store.readDerivation(drvPath),
false)).first;
}
return h->second;
}
For instance, if we change the url in a call to the `fetchurl'
function, we do not want to rebuild everything depending on it
(after all, (the hash of) the file being downloaded is unchanged).
So the *output paths* should not change. On the other hand, the
*derivation paths* should change to reflect the new dependency
graph.
/* See the header for interface details. These are the implementation details.
That's what this function does: it returns a hash which is just the
hash of the derivation ATerm, except that any input derivation
paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return a hash of
its output path. */
Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
For fixed-output derivations, each hash in the map is not the
corresponding output's content hash, but a hash of that hash along
with other constant data. The key point is that the value is a pure
function of the output's contents, and there are no preimage attacks
either spoofing an output's contents for a derivation, or
spoofing a derivation for an output's contents.
For regular derivations, it looks up each subderivation from its hash
and recurs. If the subderivation is also regular, it simply
substitutes the derivation path with its hash. If the subderivation
is fixed-output, however, it takes each output hash and pretends it
is a derivation hash producing a single "out" output. This is so we
don't leak the provenance of fixed outputs, reducing pointless cache
misses as the build itself won't know this.
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
/* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
auto hash = std::get<DerivationOutputFixed>(i->second.output);
return hashString(htSHA256, "fixed:out:"
+ hash.hash.printMethodAlgo() + ":"
+ hash.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path(store, drv.name)));
switch (drv.type()) {
case DerivationType::CAFloating:
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.path(store, drv.name)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return outputHashes;
}
case DerivationType::InputAddressed:
break;
}
/* For other derivations, replace the inputs paths with recursive
calls to this function.*/
calls to this function. */
std::map<std::string, StringSet> inputs2;
for (auto & i : drv.inputDrvs) {
auto h = drvHashes.find(i.first);
if (h == drvHashes.end()) {
assert(store.isValidPath(i.first));
h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store,
store.readDerivation(i.first), false)).first;
}
inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
const auto & res = pathDerivationModulo(store, i.first);
std::visit(overloaded {
// Regular non-CA derivation, replace derivation
[&](Hash drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
},
// CA derivation's output hashes
[&](CaOutputHashes outputHashes) {
std::set<std::string> justOut = { "out" };
for (auto & output : i.second) {
/* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output);
inputs2.insert_or_assign(
h.to_string(Base16, false),
justOut);
}
},
}, res);
}
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@ -431,14 +551,22 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
hashAlgo = string(hashAlgo, 2);
}
auto hashType = parseHashType(hashAlgo);
return DerivationOutput {
.output = DerivationOutputFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
},
}
};
return hash != ""
? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
@ -498,12 +626,19 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
for (auto & i : drv.outputs) {
out << i.first
<< store.printStorePath(i.second.path(store, drv.name));
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
out << hash->hash.printMethodAlgo()
<< hash->hash.hash.to_string(Base16, false);
} else {
out << "" << "";
}
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
out << "" << "";
},
[&](DerivationOutputCAFixed dof) {
out << dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false);
},
[&](DerivationOutputCAFloating dof) {
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
}, i.second.output);
}
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;

90
src/libstore/derivations.hh
View file

@ -6,6 +6,7 @@
#include "content-address.hh"
#include <map>
#include <variant>
namespace nix {
@ -13,20 +14,46 @@ namespace nix {
/* Abstract syntax of derivations. */
/* The traditional non-fixed-output derivation type. */
struct DerivationOutputInputAddressed
{
/* Will need to become `std::optional<StorePath>` once input-addressed
derivations are allowed to depend on cont-addressed derivations */
StorePath path;
};
struct DerivationOutputFixed
/* Fixed-output derivations, whose output paths are content addressed
according to that fixed output. */
struct DerivationOutputCAFixed
{
FixedOutputHash hash; /* hash used for expected hash computation */
};
/* Floating-output derivations, whose output paths are content addressed, but
not fixed, and so are dynamically calculated from whatever the output ends
up being. */
struct DerivationOutputCAFloating
{
/* information used for expected hash computation */
FileIngestionMethod method;
HashType hashType;
};
struct DerivationOutput
{
std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output;
StorePath path(const Store & store, std::string_view drvName) const;
std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating
> output;
std::optional<HashType> hashAlgoOpt(const Store & store) const;
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
/* DEPRECATED: Remove after CA drvs are fully implemented */
StorePath path(const Store & store, std::string_view drvName) const {
auto p = pathOpt(store, drvName);
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
return *p;
}
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -37,6 +64,25 @@ typedef std::map<StorePath, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs;
enum struct DerivationType : uint8_t {
InputAddressed,
CAFixed,
CAFloating,
};
/* Do the outputs of the derivation have paths calculated from their content,
or from the derivation itself? */
bool derivationIsCA(DerivationType);
/* Is the content of the outputs fixed a-priori via a hash? Never true for
non-CA derivations. */
bool derivationIsFixed(DerivationType);
/* Is the derivation impure and needs to access non-deterministic resources, or
pure and can be sandboxed? Note that whether or not we actually sandbox the
derivation is controlled separately. Never true for non-CA derivations. */
bool derivationIsImpure(DerivationType);
struct BasicDerivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
@ -53,7 +99,7 @@ struct BasicDerivation
bool isBuiltin() const;
/* Return true iff this is a fixed-output derivation. */
bool isFixedOutput() const;
DerivationType type() const;
/* Return the output paths of a derivation. */
StorePathSet outputPaths(const Store & store) const;
@ -90,10 +136,42 @@ Derivation readDerivation(const Store & store, const Path & drvPath, std::string
// FIXME: remove
bool isDerivation(const string & fileName);
Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
// known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
typedef std::variant<
Hash, // regular DRV normalized hash
CaOutputHashes
> DrvHashModulo;
/* Returns hashes with the details of fixed-output subderivations
expunged.
A fixed-output derivation is a derivation whose outputs have a
specified content hash and hash algorithm. (Currently they must have
exactly one output (`out'), which is specified using the `outputHash'
and `outputHashAlgo' attributes, but the algorithm doesn't assume
this.) We don't want changes to such derivations to propagate upwards
through the dependency graph, changing output paths everywhere.
For instance, if we change the url in a call to the `fetchurl'
function, we do not want to rebuild everything depending on it---after
all, (the hash of) the file being downloaded is unchanged. So the
*output paths* should not change. On the other hand, the *derivation
paths* should change to reflect the new dependency graph.
For fixed-output derivations, this returns a map from the name of
each output to its hash, unique up to the output's contents.
For regular derivations, it returns a single hash of the derivation
ATerm, after subderivations have been likewise expunged from that
derivation.
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, Hash> DrvHashes;
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe

2
src/libstore/legacy-ssh-store.cc
View file

@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
info->narHash = s.empty() ? std::optional<Hash>{} : Hash{s};
info->narHash = s.empty() ? std::optional<Hash>{} : Hash::parseAnyPrefixed(s);
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}

43
src/libstore/local-store.cc
View file

@ -544,11 +544,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
std::string drvName(drvPath.name());
drvName = string(drvName, 0, drvName.size() - drvExtension.size());
auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName)
auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
{
if (actual != expected)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(actual), printStorePath(expected));
auto j = drv.env.find(varName);
if (j == drv.env.end() || parseStorePath(j->second) != actual)
throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
@ -556,16 +553,34 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
};
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator out = drv.outputs.find("out");
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
}
// Don't need the answer, but do this anyways to assert is proper
// combination. The code below is more general and naturally allows
// combinations that are currently prohibited.
drv.type();
else {
Hash h = hashDerivationModulo(*this, drv, true);
for (auto & i : drv.outputs)
check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
std::optional<Hash> h;
for (auto & i : drv.outputs) {
std::visit(overloaded {
[&](DerivationOutputInputAddressed doia) {
if (!h) {
// somewhat expensive so we do lazily
auto temp = hashDerivationModulo(*this, drv, true);
h = std::get<Hash>(temp);
}
StorePath recomputed = makeOutputPath(i.first, *h, drvName);
if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
[&](DerivationOutputCAFixed dof) {
StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
envHasRightPath(path, i.first);
},
[&](DerivationOutputCAFloating _) {
throw UnimplementedError("floating CA output derivations are not yet implemented");
},
}, i.second.output);
}
}
@ -640,7 +655,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->id = useQueryPathInfo.getInt(0);
try {
info->narHash = Hash(useQueryPathInfo.getStr(1));
info->narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
} catch (BadHash & e) {
throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what());
}

53
src/libstore/misc.cc
View file

@ -4,6 +4,7 @@
#include "local-store.hh"
#include "store-api.hh"
#include "thread-pool.hh"
#include "topo-sort.hh"
namespace nix {
@ -112,7 +113,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
{
auto out = drv.outputs.find("out");
if (out != drv.outputs.end()) {
if (auto v = std::get_if<DerivationOutputFixed>(&out->second.output))
if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output))
return v->hash;
}
return std::nullopt;
@ -256,41 +257,21 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePaths Store::topoSortPaths(const StorePathSet & paths)
{
StorePaths sorted;
StorePathSet visited, parents;
std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit;
dfsVisit = [&](const StorePath & path, const StorePath * parent) {
if (parents.count(path))
throw BuildError("cycle detected in the references of '%s' from '%s'",
printStorePath(path), printStorePath(*parent));
if (!visited.insert(path).second) return;
parents.insert(path);
StorePathSet references;
try {
references = queryPathInfo(path)->references;
} catch (InvalidPath &) {
}
for (auto & i : references)
/* Don't traverse into paths that don't exist. That can
happen due to substitutes for non-existent paths. */
if (i != path && paths.count(i))
dfsVisit(i, &path);
sorted.push_back(path);
parents.erase(path);
};
for (auto & i : paths)
dfsVisit(i, nullptr);
std::reverse(sorted.begin(), sorted.end());
return sorted;
return topoSort(paths,
{[&](const StorePath & path) {
StorePathSet references;
try {
references = queryPathInfo(path)->references;
} catch (InvalidPath &) {
}
return references;
}},
{[&](const StorePath & path, const StorePath & parent) {
return BuildError(
"cycle detected in the references of '%s' from '%s'",
printStorePath(path),
printStorePath(parent));
}});
}

4
src/libstore/nar-info-disk-cache.cc
View file

@ -193,9 +193,9 @@ public:
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4))
narInfo->fileHash = Hash(queryNAR.getStr(4));
narInfo->fileHash = Hash::parseAnyPrefixed(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5);
narInfo->narHash = Hash(queryNAR.getStr(6));
narInfo->narHash = Hash::parseAnyPrefixed(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
narInfo->references.insert(StorePath(r));

2
src/libstore/nar-info.cc
View file

@ -12,7 +12,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
auto parseHashField = [&](const string & s) {
try {
return Hash(s);
return Hash::parseAnyPrefixed(s);
} catch (BadHash &) {
throw corrupt();
}

5
src/libstore/parsed-derivations.cc
View file

@ -117,9 +117,4 @@ bool ParsedDerivation::substitutesAllowed() const
return getBoolAttr("allowSubstitutes", true);
}
bool ParsedDerivation::contentAddressed() const
{
return getBoolAttr("__contentAddressed", false);
}
}

2
src/libstore/parsed-derivations.hh
View file

@ -34,8 +34,6 @@ public:
bool willBuildLocally() const;
bool substitutesAllowed() const;
bool contentAddressed() const;
};
}

111
src/libstore/path-info.hh Normal file
View file

@ -0,0 +1,111 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include "content-address.hh"
#include <string>
#include <optional>
namespace nix {
class Store;
struct SubstitutablePathInfo
{
std::optional<StorePath> deriver;
StorePathSet references;
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
// TODO document this
std::optional<Hash> narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
/* Whether the path is ultimately trusted, that is, it's a
derivation output that was built locally. */
bool ultimate = false;
StringSet sigs; // note: not necessarily verified
/* If non-empty, an assertion that the path is content-addressed,
i.e., that the store path is computed from a cryptographic hash
of the contents of the path, plus some other bits of data like
the "name" part of the path. Such a path doesn't need
signatures, since we don't have to trust anybody's claim that
the path is the output of a particular derivation. (In the
extensional store model, we have to trust that the *contents*
of an output path of a derivation were actually produced by
that derivation. In the intensional model, we have to trust
that a particular output path was produced by a derivation; the
path then implies the contents.)
Ideally, the content-addressability assertion would just be a Boolean,
and the store path would be computed from the name component, ‘narHash’
and ‘references’. However, we support many types of content addresses.
*/
std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
return
path == i.path
&& narHash == i.narHash
&& references == i.references;
}
/* Return a fingerprint of the store path to be used in binary
cache signatures. It contains the store path, the base-32
SHA-256 hash of the NAR serialisation of the path, the size of
the NAR, and the sorted references. The size field is strictly
speaking superfluous, but might prevent endless/excessive data
attacks. */
std::string fingerprint(const Store & store) const;
void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
/* Functions to view references + hasSelfReference as one set, mainly for
compatibility's sake. */
StorePathSet referencesPossiblyToSelf() const;
void insertReferencePossiblyToSelf(StorePath && ref);
void setReferencesPossiblyToSelf(StorePathSet && refs);
static const size_t maxSigs = std::numeric_limits<size_t>::max();
/* Return the number of signatures on this .narinfo that were
produced by one of the specified keys, or maxSigs if the path
is content-addressed. */
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
typedef list<ValidPathInfo> ValidPathInfos;
}

2
src/libstore/remote-store.cc
View file

@ -422,7 +422,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
info = std::make_shared<ValidPathInfo>(StorePath(path));
auto deriver = readString(conn->from);
if (deriver != "") info->deriver = parseStorePath(deriver);
info->narHash = Hash(readString(conn->from), htSHA256);
info->narHash = Hash::parseAny(readString(conn->from), htSHA256);
info->references = readStorePaths<StorePathSet>(*this, conn->from);
conn->from >> info->registrationTime >> info->narSize;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {

9
src/libstore/s3-binary-cache-store.cc
View file

@ -266,6 +266,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
const std::string & mimeType,
const std::string & contentEncoding)
{
istream->seekg(0, istream->end);
auto size = istream->tellg();
istream->seekg(0, istream->beg);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@ -343,10 +347,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
printInfo("uploaded 's3://%s/%s' in %d ms",
bucketName, path, duration);
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
stats.putTimeMs += duration;
stats.putBytes += std::max(size, (decltype(size)) 0);
stats.put++;
}

1
src/libstore/s3-binary-cache-store.hh
View file

@ -19,6 +19,7 @@ public:
struct Stats
{
std::atomic<uint64_t> put{0};
std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0};

6
src/libstore/store-api.cc
View file

@ -193,10 +193,6 @@ StorePath Store::makeFixedOutputPath(
}
}
// FIXME Put this somewhere?
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
const StorePathSet & references, bool hasSelfReference) const
{
@ -876,7 +872,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
if (hashGiven) {
string s;
getline(str, s);
info.narHash = Hash(s, htSHA256);
info.narHash = Hash::parseAny(s, htSHA256);
getline(str, s);
if (!string2Int(s, info.narSize)) throw Error("number expected");
}

90
src/libstore/store-api.hh
View file

@ -10,6 +10,7 @@
#include "globals.hh"
#include "config.hh"
#include "derivations.hh"
#include "path-info.hh"
#include <atomic>
#include <limits>
@ -101,95 +102,6 @@ struct GCResults
};
struct SubstitutablePathInfo
{
std::optional<StorePath> deriver;
StorePathSet references;
uint64_t downloadSize; /* 0 = unknown or inapplicable */
uint64_t narSize; /* 0 = unknown */
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
// TODO document this
std::optional<Hash> narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
/* Whether the path is ultimately trusted, that is, it's a
derivation output that was built locally. */
bool ultimate = false;
StringSet sigs; // note: not necessarily verified
/* If non-empty, an assertion that the path is content-addressed,
i.e., that the store path is computed from a cryptographic hash
of the contents of the path, plus some other bits of data like
the "name" part of the path. Such a path doesn't need
signatures, since we don't have to trust anybody's claim that
the path is the output of a particular derivation. (In the
extensional store model, we have to trust that the *contents*
of an output path of a derivation were actually produced by
that derivation. In the intensional model, we have to trust
that a particular output path was produced by a derivation; the
path then implies the contents.)
Ideally, the content-addressability assertion would just be a Boolean,
and the store path would be computed from the name component, ‘narHash’
and ‘references’. However, we support many types of content addresses.
*/
std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
return
path == i.path
&& narHash == i.narHash
&& references == i.references;
}
/* Return a fingerprint of the store path to be used in binary
cache signatures. It contains the store path, the base-32
SHA-256 hash of the NAR serialisation of the path, the size of
the NAR, and the sorted references. The size field is strictly
speaking superfluous, but might prevent endless/excessive data
attacks. */
std::string fingerprint(const Store & store) const;
void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
static const size_t maxSigs = std::numeric_limits<size_t>::max();
/* Return the number of signatures on this .narinfo that were
produced by one of the specified keys, or maxSigs if the path
is content-addressed. */
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
typedef list<ValidPathInfo> ValidPathInfos;
enum BuildMode { bmNormal, bmRepair, bmCheck };