mirror of
https://github.com/NixOS/nix.git
synced 2025-11-16 07:22:43 +01:00
Merge branch 'master' into angerman/mac-fix-recursive-nix
This commit is contained in:
Eelco Dolstra
GitHub
commit
381a32981b
229 changed files with 4173 additions and 1916 deletions
|
|
@ -274,11 +274,13 @@ void DerivationGoal::haveDerivation()
|
|||
)
|
||||
)
|
||||
);
|
||||
else
|
||||
else {
|
||||
auto * cap = getDerivationCA(*drv);
|
||||
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
|
||||
status.known->path,
|
||||
buildMode == bmRepair ? Repair : NoRepair,
|
||||
getDerivationCA(*drv))));
|
||||
cap ? std::optional { *cap } : std::nullopt)));
|
||||
}
|
||||
}
|
||||
|
||||
if (waitees.empty()) /* to prevent hang (no wake-up event) */
|
||||
|
|
@ -1020,43 +1022,33 @@ void DerivationGoal::resolvedFinished()
|
|||
|
||||
StorePathSet outputPaths;
|
||||
|
||||
// `wantedOutputs` might merely indicate “all the outputs”
|
||||
auto realWantedOutputs = std::visit(overloaded {
|
||||
[&](const OutputsSpec::All &) {
|
||||
return resolvedDrv.outputNames();
|
||||
},
|
||||
[&](const OutputsSpec::Names & names) {
|
||||
return static_cast<std::set<std::string>>(names);
|
||||
},
|
||||
}, wantedOutputs.raw());
|
||||
|
||||
for (auto & wantedOutput : realWantedOutputs) {
|
||||
auto initialOutput = get(initialOutputs, wantedOutput);
|
||||
auto resolvedHash = get(resolvedHashes, wantedOutput);
|
||||
for (auto & outputName : resolvedDrv.outputNames()) {
|
||||
auto initialOutput = get(initialOutputs, outputName);
|
||||
auto resolvedHash = get(resolvedHashes, outputName);
|
||||
if ((!initialOutput) || (!resolvedHash))
|
||||
throw Error(
|
||||
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
|
||||
worker.store.printStorePath(drvPath), wantedOutput);
|
||||
worker.store.printStorePath(drvPath), outputName);
|
||||
|
||||
auto realisation = [&]{
|
||||
auto take1 = get(resolvedResult.builtOutputs, wantedOutput);
|
||||
auto take1 = get(resolvedResult.builtOutputs, outputName);
|
||||
if (take1) return *take1;
|
||||
|
||||
/* The above `get` should work. But sateful tracking of
|
||||
outputs in resolvedResult, this can get out of sync with the
|
||||
store, which is our actual source of truth. For now we just
|
||||
check the store directly if it fails. */
|
||||
auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput });
|
||||
auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName });
|
||||
if (take2) return *take2;
|
||||
|
||||
throw Error(
|
||||
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
|
||||
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
|
||||
worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
|
||||
}();
|
||||
|
||||
if (drv->type().isPure()) {
|
||||
auto newRealisation = realisation;
|
||||
newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
|
||||
newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
|
||||
newRealisation.signatures.clear();
|
||||
if (!drv->type().isFixed())
|
||||
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
|
||||
|
|
@ -1064,7 +1056,7 @@ void DerivationGoal::resolvedFinished()
|
|||
worker.store.registerDrvOutput(newRealisation);
|
||||
}
|
||||
outputPaths.insert(realisation.outPath);
|
||||
builtOutputs.emplace(wantedOutput, realisation);
|
||||
builtOutputs.emplace(outputName, realisation);
|
||||
}
|
||||
|
||||
runPostBuildHook(
|
||||
|
|
@ -1160,7 +1152,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
|
||||
/* Tell the hook all the inputs that have to be copied to the
|
||||
remote system. */
|
||||
worker_proto::write(worker.store, hook->sink, inputPaths);
|
||||
workerProtoWrite(worker.store, hook->sink, inputPaths);
|
||||
|
||||
/* Tell the hooks the missing outputs that have to be copied back
|
||||
from the remote system. */
|
||||
|
|
@ -1171,7 +1163,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
|
||||
missingOutputs.insert(outputName);
|
||||
}
|
||||
worker_proto::write(worker.store, hook->sink, missingOutputs);
|
||||
workerProtoWrite(worker.store, hook->sink, missingOutputs);
|
||||
}
|
||||
|
||||
hook->sink = FdSink();
|
||||
|
|
@ -1406,7 +1398,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
|
|||
);
|
||||
}
|
||||
}
|
||||
if (info.wanted && info.known && info.known->isValid())
|
||||
if (info.known && info.known->isValid())
|
||||
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
|
||||
}
|
||||
|
||||
|
|
@ -1457,8 +1449,9 @@ void DerivationGoal::done(
|
|||
mcRunningBuilds.reset();
|
||||
|
||||
if (buildResult.success()) {
|
||||
assert(!builtOutputs.empty());
|
||||
buildResult.builtOutputs = std::move(builtOutputs);
|
||||
auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
|
||||
assert(!wantedBuiltOutputs.empty());
|
||||
buildResult.builtOutputs = std::move(wantedBuiltOutputs);
|
||||
if (status == BuildResult::Built)
|
||||
worker.doneBuilds++;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -306,15 +306,13 @@ struct DerivationGoal : public Goal
|
|||
* Update 'initialOutputs' to determine the current status of the
|
||||
* outputs of the derivation. Also returns a Boolean denoting
|
||||
* whether all outputs are valid and non-corrupt, and a
|
||||
* 'SingleDrvOutputs' structure containing the valid and wanted
|
||||
* outputs.
|
||||
* 'SingleDrvOutputs' structure containing the valid outputs.
|
||||
*/
|
||||
std::pair<bool, SingleDrvOutputs> checkPathValidity();
|
||||
|
||||
/**
|
||||
* Aborts if any output is not valid or corrupt, and otherwise
|
||||
* returns a 'SingleDrvOutputs' structure containing the wanted
|
||||
* outputs.
|
||||
* returns a 'SingleDrvOutputs' structure containing all outputs.
|
||||
*/
|
||||
SingleDrvOutputs assertPathValidity();
|
||||
|
||||
|
|
@ -335,6 +333,8 @@ struct DerivationGoal : public Goal
|
|||
void waiteeDone(GoalPtr waitee, ExitCode result) override;
|
||||
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||
|
||||
JobCategory jobCategory() override { return JobCategory::Build; };
|
||||
};
|
||||
|
||||
MakeError(NotDeterministic, BuildError);
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ class Worker;
|
|||
class DrvOutputSubstitutionGoal : public Goal {
|
||||
|
||||
/**
|
||||
* The drv output we're trying to substitue
|
||||
* The drv output we're trying to substitute
|
||||
*/
|
||||
DrvOutput id;
|
||||
|
||||
|
|
@ -72,6 +72,8 @@ public:
|
|||
|
||||
void work() override;
|
||||
void handleEOF(int fd) override;
|
||||
|
||||
JobCategory jobCategory() override { return JobCategory::Substitution; };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ void Store::ensurePath(const StorePath & path)
|
|||
}
|
||||
|
||||
|
||||
void LocalStore::repairPath(const StorePath & path)
|
||||
void Store::repairPath(const StorePath & path)
|
||||
{
|
||||
Worker worker(*this, *this);
|
||||
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
|
||||
|
|
|
|||
|
|
@ -34,6 +34,17 @@ typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
|
|||
*/
|
||||
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
|
||||
|
||||
/**
|
||||
* Used as a hint to the worker on how to schedule a particular goal. For example,
|
||||
* builds are typically CPU- and memory-bound, while substitutions are I/O bound.
|
||||
* Using this information, the worker might decide to schedule more or fewer goals
|
||||
* of each category in parallel.
|
||||
*/
|
||||
enum struct JobCategory {
|
||||
Build,
|
||||
Substitution,
|
||||
};
|
||||
|
||||
struct Goal : public std::enable_shared_from_this<Goal>
|
||||
{
|
||||
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
|
||||
|
|
@ -150,6 +161,8 @@ public:
|
|||
void amDone(ExitCode result, std::optional<Error> ex = {});
|
||||
|
||||
virtual void cleanup() { }
|
||||
|
||||
virtual JobCategory jobCategory() = 0;
|
||||
};
|
||||
|
||||
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
|
||||
|
|
|
|||
|
|
@ -357,7 +357,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
|
|||
for (auto & [_, status] : initialOutputs) {
|
||||
if (!status.known) continue;
|
||||
if (buildMode != bmCheck && status.known->isValid()) continue;
|
||||
auto p = worker.store.printStorePath(status.known->path);
|
||||
auto p = worker.store.toRealPath(status.known->path);
|
||||
if (pathExists(chrootRootDir + p))
|
||||
renameFile((chrootRootDir + p), p);
|
||||
}
|
||||
|
|
@ -1791,6 +1791,9 @@ void LocalDerivationGoal::runChild()
|
|||
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
|
||||
if (pathExists(path))
|
||||
ss.push_back(path);
|
||||
|
||||
if (settings.caFile != "")
|
||||
dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
|
||||
}
|
||||
|
||||
for (auto & i : ss) dirsInChroot.emplace(i, i);
|
||||
|
|
@ -2441,37 +2444,51 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
throw BuildError(
|
||||
"output path %1% without valid stats info",
|
||||
actualPath);
|
||||
if (outputHash.method == FileIngestionMethod::Flat) {
|
||||
if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
|
||||
outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
|
||||
{
|
||||
/* The output path should be a regular file without execute permission. */
|
||||
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
|
||||
throw BuildError(
|
||||
"output path '%1%' should be a non-executable regular file "
|
||||
"since recursive hashing is not enabled (outputHashMode=flat)",
|
||||
"since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
|
||||
actualPath);
|
||||
}
|
||||
rewriteOutput();
|
||||
/* FIXME optimize and deduplicate with addToStore */
|
||||
std::string oldHashPart { scratchPath->hashPart() };
|
||||
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
||||
switch (outputHash.method) {
|
||||
case FileIngestionMethod::Recursive:
|
||||
dumpPath(actualPath, caSink);
|
||||
break;
|
||||
case FileIngestionMethod::Flat:
|
||||
readFile(actualPath, caSink);
|
||||
break;
|
||||
}
|
||||
std::visit(overloaded {
|
||||
[&](const TextIngestionMethod &) {
|
||||
readFile(actualPath, caSink);
|
||||
},
|
||||
[&](const FileIngestionMethod & m2) {
|
||||
switch (m2) {
|
||||
case FileIngestionMethod::Recursive:
|
||||
dumpPath(actualPath, caSink);
|
||||
break;
|
||||
case FileIngestionMethod::Flat:
|
||||
readFile(actualPath, caSink);
|
||||
break;
|
||||
}
|
||||
},
|
||||
}, outputHash.method.raw);
|
||||
auto got = caSink.finish().first;
|
||||
|
||||
auto optCA = ContentAddressWithReferences::fromPartsOpt(
|
||||
outputHash.method,
|
||||
std::move(got),
|
||||
rewriteRefs());
|
||||
if (!optCA) {
|
||||
// TODO track distinct failure modes separately (at the time of
|
||||
// writing there is just one but `nullopt` is unclear) so this
|
||||
// message can't get out of sync.
|
||||
throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
|
||||
}
|
||||
ValidPathInfo newInfo0 {
|
||||
worker.store,
|
||||
outputPathName(drv->name, outputName),
|
||||
FixedOutputInfo {
|
||||
.hash = {
|
||||
.method = outputHash.method,
|
||||
.hash = got,
|
||||
},
|
||||
.references = rewriteRefs(),
|
||||
},
|
||||
*std::move(optCA),
|
||||
Hash::dummy,
|
||||
};
|
||||
if (*scratchPath != newInfo0.path) {
|
||||
|
|
@ -2518,13 +2535,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
},
|
||||
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
auto wanted = dof.ca.getHash();
|
||||
|
||||
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
|
||||
.method = dof.hash.method,
|
||||
.hashType = dof.hash.hash.type,
|
||||
.method = dof.ca.getMethod(),
|
||||
.hashType = wanted.type,
|
||||
});
|
||||
|
||||
/* Check wanted hash */
|
||||
const Hash & wanted = dof.hash.hash;
|
||||
assert(newInfo0.ca);
|
||||
auto got = newInfo0.ca->getHash();
|
||||
if (wanted != got) {
|
||||
|
|
@ -2537,6 +2555,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
wanted.to_string(SRI, true),
|
||||
got.to_string(SRI, true)));
|
||||
}
|
||||
if (!newInfo0.references.empty())
|
||||
delayedException = std::make_exception_ptr(
|
||||
BuildError("illegal path references in fixed-output derivation '%s'",
|
||||
worker.store.printStorePath(drvPath)));
|
||||
|
||||
return newInfo0;
|
||||
},
|
||||
|
||||
|
|
@ -2716,8 +2739,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
signRealisation(thisRealisation);
|
||||
worker.store.registerDrvOutput(thisRealisation);
|
||||
}
|
||||
if (wantedOutputs.contains(outputName))
|
||||
builtOutputs.emplace(outputName, thisRealisation);
|
||||
builtOutputs.emplace(outputName, thisRealisation);
|
||||
}
|
||||
|
||||
return builtOutputs;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ void setPersonality(std::string_view system)
|
|||
&& (std::string_view(SYSTEM) == "x86_64-linux"
|
||||
|| (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
|
||||
|| system == "armv7l-linux"
|
||||
|| system == "armv6l-linux")
|
||||
|| system == "armv6l-linux"
|
||||
|| system == "armv5tel-linux")
|
||||
{
|
||||
if (personality(PER_LINUX32) == -1)
|
||||
throw SysError("cannot set 32-bit personality");
|
||||
|
|
|
|||
|
|
@ -200,11 +200,10 @@ void PathSubstitutionGoal::tryToRun()
|
|||
{
|
||||
trace("trying to run");
|
||||
|
||||
/* Make sure that we are allowed to start a build. Note that even
|
||||
if maxBuildJobs == 0 (no local builds allowed), we still allow
|
||||
a substituter to run. This is because substitutions cannot be
|
||||
distributed to another machine via the build hook. */
|
||||
if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
|
||||
/* Make sure that we are allowed to start a substitution. Note that even
|
||||
if maxSubstitutionJobs == 0, we still allow a substituter to run. This
|
||||
prevents infinite waiting. */
|
||||
if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
|
||||
worker.waitForBuildSlot(shared_from_this());
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,6 +115,8 @@ public:
|
|||
void handleEOF(int fd) override;
|
||||
|
||||
void cleanup() override;
|
||||
|
||||
JobCategory jobCategory() override { return JobCategory::Substitution; };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ Worker::Worker(Store & store, Store & evalStore)
|
|||
{
|
||||
/* Debugging: prevent recursive workers. */
|
||||
nrLocalBuilds = 0;
|
||||
nrSubstitutions = 0;
|
||||
lastWokenUp = steady_time_point::min();
|
||||
permanentFailure = false;
|
||||
timedOut = false;
|
||||
|
|
@ -176,6 +177,12 @@ unsigned Worker::getNrLocalBuilds()
|
|||
}
|
||||
|
||||
|
||||
unsigned Worker::getNrSubstitutions()
|
||||
{
|
||||
return nrSubstitutions;
|
||||
}
|
||||
|
||||
|
||||
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
|
||||
bool inBuildSlot, bool respectTimeouts)
|
||||
{
|
||||
|
|
@ -187,7 +194,10 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
|
|||
child.inBuildSlot = inBuildSlot;
|
||||
child.respectTimeouts = respectTimeouts;
|
||||
children.emplace_back(child);
|
||||
if (inBuildSlot) nrLocalBuilds++;
|
||||
if (inBuildSlot) {
|
||||
if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
|
||||
else nrLocalBuilds++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -198,8 +208,13 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
|
|||
if (i == children.end()) return;
|
||||
|
||||
if (i->inBuildSlot) {
|
||||
assert(nrLocalBuilds > 0);
|
||||
nrLocalBuilds--;
|
||||
if (goal->jobCategory() == JobCategory::Substitution) {
|
||||
assert(nrSubstitutions > 0);
|
||||
nrSubstitutions--;
|
||||
} else {
|
||||
assert(nrLocalBuilds > 0);
|
||||
nrLocalBuilds--;
|
||||
}
|
||||
}
|
||||
|
||||
children.erase(i);
|
||||
|
|
@ -220,7 +235,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
|
|||
void Worker::waitForBuildSlot(GoalPtr goal)
|
||||
{
|
||||
debug("wait for build slot");
|
||||
if (getNrLocalBuilds() < settings.maxBuildJobs)
|
||||
bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
|
||||
if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
|
||||
(isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
|
||||
wakeUp(goal); /* we can do it right away */
|
||||
else
|
||||
addToWeakGoals(wantingToBuild, goal);
|
||||
|
|
|
|||
|
|
@ -88,11 +88,16 @@ private:
|
|||
std::list<Child> children;
|
||||
|
||||
/**
|
||||
* Number of build slots occupied. This includes local builds and
|
||||
* substitutions but not remote builds via the build hook.
|
||||
* Number of build slots occupied. This includes local builds but does not
|
||||
* include substitutions or remote builds via the build hook.
|
||||
*/
|
||||
unsigned int nrLocalBuilds;
|
||||
|
||||
/**
|
||||
* Number of substitution slots occupied.
|
||||
*/
|
||||
unsigned int nrSubstitutions;
|
||||
|
||||
/**
|
||||
* Maps used to prevent multiple instantiations of a goal for the
|
||||
* same derivation / path.
|
||||
|
|
@ -220,12 +225,16 @@ public:
|
|||
void wakeUp(GoalPtr goal);
|
||||
|
||||
/**
|
||||
* Return the number of local build and substitution processes
|
||||
* currently running (but not remote builds via the build
|
||||
* hook).
|
||||
* Return the number of local build processes currently running (but not
|
||||
* remote builds via the build hook).
|
||||
*/
|
||||
unsigned int getNrLocalBuilds();
|
||||
|
||||
/**
|
||||
* Return the number of substitution processes currently running.
|
||||
*/
|
||||
unsigned int getNrSubstitutions();
|
||||
|
||||
/**
|
||||
* Registers a running child process. `inBuildSlot` means that
|
||||
* the process counts towards the jobs limit.
|
||||
|
|
|
|||
|
|
@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
|
|||
}
|
||||
}
|
||||
|
||||
std::string ContentAddressMethod::renderPrefix() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](TextIngestionMethod) -> std::string { return "text:"; },
|
||||
[](FileIngestionMethod m2) {
|
||||
/* Not prefixed for back compat with things that couldn't produce text before. */
|
||||
return makeFileIngestionPrefix(m2);
|
||||
},
|
||||
}, raw);
|
||||
}
|
||||
|
||||
ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
|
||||
{
|
||||
ContentAddressMethod method = FileIngestionMethod::Flat;
|
||||
if (splitPrefix(m, "r:"))
|
||||
method = FileIngestionMethod::Recursive;
|
||||
else if (splitPrefix(m, "text:"))
|
||||
method = TextIngestionMethod {};
|
||||
return method;
|
||||
}
|
||||
|
||||
std::string ContentAddress::render() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
|
|
@ -36,14 +57,14 @@ std::string ContentAddress::render() const
|
|||
}, raw);
|
||||
}
|
||||
|
||||
std::string ContentAddressMethod::render() const
|
||||
std::string ContentAddressMethod::render(HashType ht) const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const TextHashMethod & th) {
|
||||
return std::string{"text:"} + printHashType(htSHA256);
|
||||
[&](const TextIngestionMethod & th) {
|
||||
return std::string{"text:"} + printHashType(ht);
|
||||
},
|
||||
[](const FixedOutputHashMethod & fshm) {
|
||||
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
|
||||
[&](const FileIngestionMethod & fim) {
|
||||
return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
|
||||
}
|
||||
}, raw);
|
||||
}
|
||||
|
|
@ -51,7 +72,7 @@ std::string ContentAddressMethod::render() const
|
|||
/**
|
||||
* Parses content address strings up to the hash.
|
||||
*/
|
||||
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
|
||||
static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
|
||||
{
|
||||
std::string_view wholeInput { rest };
|
||||
|
||||
|
|
@ -75,46 +96,47 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
|
|||
if (prefix == "text") {
|
||||
// No parsing of the ingestion method, "text" only support flat.
|
||||
HashType hashType = parseHashType_();
|
||||
if (hashType != htSHA256)
|
||||
throw Error("text content address hash should use %s, but instead uses %s",
|
||||
printHashType(htSHA256), printHashType(hashType));
|
||||
return TextHashMethod {};
|
||||
return {
|
||||
TextIngestionMethod {},
|
||||
std::move(hashType),
|
||||
};
|
||||
} else if (prefix == "fixed") {
|
||||
// Parse method
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
if (splitPrefix(rest, "r:"))
|
||||
method = FileIngestionMethod::Recursive;
|
||||
HashType hashType = parseHashType_();
|
||||
return FixedOutputHashMethod {
|
||||
.fileIngestionMethod = method,
|
||||
.hashType = std::move(hashType),
|
||||
return {
|
||||
std::move(method),
|
||||
std::move(hashType),
|
||||
};
|
||||
} else
|
||||
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
|
||||
}
|
||||
|
||||
ContentAddress ContentAddress::parse(std::string_view rawCa) {
|
||||
ContentAddress ContentAddress::parse(std::string_view rawCa)
|
||||
{
|
||||
auto rest = rawCa;
|
||||
|
||||
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
|
||||
auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest);
|
||||
auto hashType = hashType_; // work around clang bug
|
||||
|
||||
return std::visit(
|
||||
overloaded {
|
||||
[&](TextHashMethod & thm) {
|
||||
return ContentAddress(TextHash {
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
|
||||
});
|
||||
},
|
||||
[&](FixedOutputHashMethod & fohMethod) {
|
||||
return ContentAddress(FixedOutputHash {
|
||||
.method = fohMethod.fileIngestionMethod,
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
|
||||
});
|
||||
},
|
||||
}, caMethod.raw);
|
||||
return std::visit(overloaded {
|
||||
[&](TextIngestionMethod &) {
|
||||
return ContentAddress(TextHash {
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, hashType)
|
||||
});
|
||||
},
|
||||
[&](FileIngestionMethod & fim) {
|
||||
return ContentAddress(FixedOutputHash {
|
||||
.method = fim,
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, hashType),
|
||||
});
|
||||
},
|
||||
}, caMethod.raw);
|
||||
}
|
||||
|
||||
ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod)
|
||||
std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
|
||||
{
|
||||
std::string asPrefix = std::string{caMethod} + ":";
|
||||
// parseContentAddressMethodPrefix takes its argument by reference
|
||||
|
|
@ -134,6 +156,36 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
|
|||
return ca ? ca->render() : "";
|
||||
}
|
||||
|
||||
ContentAddress ContentAddress::fromParts(
|
||||
ContentAddressMethod method, Hash hash) noexcept
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[&](TextIngestionMethod _) -> ContentAddress {
|
||||
return TextHash {
|
||||
.hash = std::move(hash),
|
||||
};
|
||||
},
|
||||
[&](FileIngestionMethod m2) -> ContentAddress {
|
||||
return FixedOutputHash {
|
||||
.method = std::move(m2),
|
||||
.hash = std::move(hash),
|
||||
};
|
||||
},
|
||||
}, method.raw);
|
||||
}
|
||||
|
||||
ContentAddressMethod ContentAddress::getMethod() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const TextHash & th) -> ContentAddressMethod {
|
||||
return TextIngestionMethod {};
|
||||
},
|
||||
[](const FixedOutputHash & fsh) -> ContentAddressMethod {
|
||||
return fsh.method;
|
||||
},
|
||||
}, raw);
|
||||
}
|
||||
|
||||
const Hash & ContentAddress::getHash() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
|
|
@ -146,6 +198,12 @@ const Hash & ContentAddress::getHash() const
|
|||
}, raw);
|
||||
}
|
||||
|
||||
std::string ContentAddress::printMethodAlgo() const
|
||||
{
|
||||
return getMethod().renderPrefix()
|
||||
+ printHashType(getHash().type);
|
||||
}
|
||||
|
||||
bool StoreReferences::empty() const
|
||||
{
|
||||
return !self && others.empty();
|
||||
|
|
@ -156,7 +214,8 @@ size_t StoreReferences::size() const
|
|||
return (self ? 1 : 0) + others.size();
|
||||
}
|
||||
|
||||
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) {
|
||||
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[&](const TextHash & h) -> ContentAddressWithReferences {
|
||||
return TextInfo {
|
||||
|
|
@ -173,4 +232,56 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
|
|||
}, ca.raw);
|
||||
}
|
||||
|
||||
std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
|
||||
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
|
||||
if (refs.self)
|
||||
return std::nullopt;
|
||||
return ContentAddressWithReferences {
|
||||
TextInfo {
|
||||
.hash = { .hash = std::move(hash) },
|
||||
.references = std::move(refs.others),
|
||||
}
|
||||
};
|
||||
},
|
||||
[&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
|
||||
return ContentAddressWithReferences {
|
||||
FixedOutputInfo {
|
||||
.hash = {
|
||||
.method = m2,
|
||||
.hash = std::move(hash),
|
||||
},
|
||||
.references = std::move(refs),
|
||||
}
|
||||
};
|
||||
},
|
||||
}, method.raw);
|
||||
}
|
||||
|
||||
ContentAddressMethod ContentAddressWithReferences::getMethod() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const TextInfo & th) -> ContentAddressMethod {
|
||||
return TextIngestionMethod {};
|
||||
},
|
||||
[](const FixedOutputInfo & fsh) -> ContentAddressMethod {
|
||||
return fsh.hash.method;
|
||||
},
|
||||
}, raw);
|
||||
}
|
||||
|
||||
Hash ContentAddressWithReferences::getHash() const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](const TextInfo & th) {
|
||||
return th.hash.hash;
|
||||
},
|
||||
[](const FixedOutputInfo & fsh) {
|
||||
return fsh.hash.hash;
|
||||
},
|
||||
}, raw);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,14 @@ namespace nix {
|
|||
*
|
||||
* Somewhat obscure, used by \ref Derivation derivations and
|
||||
* `builtins.toFile` currently.
|
||||
*
|
||||
* TextIngestionMethod is identical to FileIngestionMethod::Fixed except that
|
||||
* the former may not have self-references and is tagged `text:${algo}:${hash}`
|
||||
* rather than `fixed:${algo}:${hash}`. The contents of the store path are
|
||||
* ingested and hashed identically, aside from the slightly different tag and
|
||||
* restriction on self-references.
|
||||
*/
|
||||
struct TextHashMethod : std::monostate { };
|
||||
struct TextIngestionMethod : std::monostate { };
|
||||
|
||||
/**
|
||||
* An enumeration of the main ways we can serialize file system
|
||||
|
|
@ -46,13 +52,6 @@ enum struct FileIngestionMethod : uint8_t {
|
|||
*/
|
||||
std::string makeFileIngestionPrefix(FileIngestionMethod m);
|
||||
|
||||
struct FixedOutputHashMethod {
|
||||
FileIngestionMethod fileIngestionMethod;
|
||||
HashType hashType;
|
||||
|
||||
GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
|
||||
};
|
||||
|
||||
/**
|
||||
* An enumeration of all the ways we can serialize file system objects.
|
||||
*
|
||||
|
|
@ -64,8 +63,8 @@ struct FixedOutputHashMethod {
|
|||
struct ContentAddressMethod
|
||||
{
|
||||
typedef std::variant<
|
||||
TextHashMethod,
|
||||
FixedOutputHashMethod
|
||||
TextIngestionMethod,
|
||||
FileIngestionMethod
|
||||
> Raw;
|
||||
|
||||
Raw raw;
|
||||
|
|
@ -77,9 +76,36 @@ struct ContentAddressMethod
|
|||
: raw(std::forward<decltype(arg)>(arg)...)
|
||||
{ }
|
||||
|
||||
static ContentAddressMethod parse(std::string_view rawCaMethod);
|
||||
|
||||
std::string render() const;
|
||||
/**
|
||||
* Parse the prefix tag which indicates how the files
|
||||
* were ingested, with the fixed output case not prefixed for back
|
||||
* compat.
|
||||
*
|
||||
* @param [in] m A string that should begin with the prefix.
|
||||
* @param [out] m The remainder of the string after the prefix.
|
||||
*/
|
||||
static ContentAddressMethod parsePrefix(std::string_view & m);
|
||||
|
||||
/**
|
||||
* Render the prefix tag which indicates how the files wre ingested.
|
||||
*
|
||||
* The rough inverse of `parsePrefix()`.
|
||||
*/
|
||||
std::string renderPrefix() const;
|
||||
|
||||
/**
|
||||
* Parse a content addressing method and hash type.
|
||||
*/
|
||||
static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
|
||||
|
||||
/**
|
||||
* Render a content addressing method and hash type in a
|
||||
* nicer way, prefixing both cases.
|
||||
*
|
||||
* The rough inverse of `parse()`.
|
||||
*/
|
||||
std::string render(HashType ht) const;
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -147,8 +173,9 @@ struct ContentAddress
|
|||
{ }
|
||||
|
||||
/**
|
||||
* Compute the content-addressability assertion (ValidPathInfo::ca) for
|
||||
* paths created by Store::makeFixedOutputPath() / Store::addToStore().
|
||||
* Compute the content-addressability assertion
|
||||
* (`ValidPathInfo::ca`) for paths created by
|
||||
* `Store::makeFixedOutputPath()` / `Store::addToStore()`.
|
||||
*/
|
||||
std::string render() const;
|
||||
|
||||
|
|
@ -156,9 +183,27 @@ struct ContentAddress
|
|||
|
||||
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
|
||||
|
||||
/**
|
||||
* Create a `ContentAddress` from 2 parts:
|
||||
*
|
||||
* @param method Way ingesting the file system data.
|
||||
*
|
||||
* @param hash Hash of ingested file system data.
|
||||
*/
|
||||
static ContentAddress fromParts(
|
||||
ContentAddressMethod method, Hash hash) noexcept;
|
||||
|
||||
ContentAddressMethod getMethod() const;
|
||||
|
||||
const Hash & getHash() const;
|
||||
|
||||
std::string printMethodAlgo() const;
|
||||
};
|
||||
|
||||
/**
|
||||
* Render the `ContentAddress` if it exists to a string, return empty
|
||||
* string otherwise.
|
||||
*/
|
||||
std::string renderContentAddress(std::optional<ContentAddress> ca);
|
||||
|
||||
|
||||
|
|
@ -244,10 +289,29 @@ struct ContentAddressWithReferences
|
|||
{ }
|
||||
|
||||
/**
|
||||
* Create a ContentAddressWithReferences from a mere ContentAddress, by
|
||||
* assuming no references in all cases.
|
||||
* Create a `ContentAddressWithReferences` from a mere
|
||||
* `ContentAddress`, by claiming no references.
|
||||
*/
|
||||
static ContentAddressWithReferences withoutRefs(const ContentAddress &);
|
||||
static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept;
|
||||
|
||||
/**
|
||||
* Create a `ContentAddressWithReferences` from 3 parts:
|
||||
*
|
||||
* @param method Way ingesting the file system data.
|
||||
*
|
||||
* @param hash Hash of ingested file system data.
|
||||
*
|
||||
* @param refs References to other store objects or oneself.
|
||||
*
|
||||
* Do note that not all combinations are supported; `nullopt` is
|
||||
* returns for invalid combinations.
|
||||
*/
|
||||
static std::optional<ContentAddressWithReferences> fromPartsOpt(
|
||||
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
|
||||
|
||||
ContentAddressMethod getMethod() const;
|
||||
|
||||
Hash getHash() const;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -263,7 +263,7 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli
|
|||
{
|
||||
std::vector<DerivedPath> reqs;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
|
||||
reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
|
||||
reqs = WorkerProto<std::vector<DerivedPath>>::read(store, from);
|
||||
} else {
|
||||
for (auto & s : readStrings<Strings>(from))
|
||||
reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
|
||||
|
|
@ -287,7 +287,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
|
||||
case wopQueryValidPaths: {
|
||||
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
auto paths = WorkerProto<StorePathSet>::read(*store, from);
|
||||
|
||||
SubstituteFlag substitute = NoSubstitute;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
|
||||
|
|
@ -300,7 +300,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
auto res = store->queryValidPaths(paths, substitute);
|
||||
logger->stopWork();
|
||||
worker_proto::write(*store, to, res);
|
||||
workerProtoWrite(*store, to, res);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -316,11 +316,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
|
||||
case wopQuerySubstitutablePaths: {
|
||||
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
auto paths = WorkerProto<StorePathSet>::read(*store, from);
|
||||
logger->startWork();
|
||||
auto res = store->querySubstitutablePaths(paths);
|
||||
logger->stopWork();
|
||||
worker_proto::write(*store, to, res);
|
||||
workerProtoWrite(*store, to, res);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -349,7 +349,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
paths = store->queryValidDerivers(path);
|
||||
else paths = store->queryDerivationOutputs(path);
|
||||
logger->stopWork();
|
||||
worker_proto::write(*store, to, paths);
|
||||
workerProtoWrite(*store, to, paths);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -367,7 +367,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
logger->startWork();
|
||||
auto outputs = store->queryPartialDerivationOutputMap(path);
|
||||
logger->stopWork();
|
||||
worker_proto::write(*store, to, outputs);
|
||||
workerProtoWrite(*store, to, outputs);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -393,7 +393,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
|
||||
auto name = readString(from);
|
||||
auto camStr = readString(from);
|
||||
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
auto refs = WorkerProto<StorePathSet>::read(*store, from);
|
||||
bool repairBool;
|
||||
from >> repairBool;
|
||||
auto repair = RepairFlag{repairBool};
|
||||
|
|
@ -401,18 +401,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
logger->startWork();
|
||||
auto pathInfo = [&]() {
|
||||
// NB: FramedSource must be out of scope before logger->stopWork();
|
||||
ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr);
|
||||
auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
|
||||
auto hashType = hashType_; // work around clang bug
|
||||
FramedSource source(from);
|
||||
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
|
||||
return std::visit(overloaded {
|
||||
[&](const TextHashMethod &) {
|
||||
[&](const TextIngestionMethod &) {
|
||||
if (hashType != htSHA256)
|
||||
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
|
||||
name, printHashType(hashType));
|
||||
// We could stream this by changing Store
|
||||
std::string contents = source.drain();
|
||||
auto path = store->addTextToStore(name, contents, refs, repair);
|
||||
return store->queryPathInfo(path);
|
||||
},
|
||||
[&](const FixedOutputHashMethod & fohm) {
|
||||
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
|
||||
[&](const FileIngestionMethod & fim) {
|
||||
auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
|
||||
return store->queryPathInfo(path);
|
||||
},
|
||||
}, contentAddressMethod.raw);
|
||||
|
|
@ -491,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
case wopAddTextToStore: {
|
||||
std::string suffix = readString(from);
|
||||
std::string s = readString(from);
|
||||
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
auto refs = WorkerProto<StorePathSet>::read(*store, from);
|
||||
logger->startWork();
|
||||
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
|
||||
logger->stopWork();
|
||||
|
|
@ -563,7 +567,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
auto results = store->buildPathsWithResults(drvs, mode);
|
||||
logger->stopWork();
|
||||
|
||||
worker_proto::write(*store, to, results);
|
||||
workerProtoWrite(*store, to, results);
|
||||
|
||||
break;
|
||||
}
|
||||
|
|
@ -640,7 +644,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
DrvOutputs builtOutputs;
|
||||
for (auto & [output, realisation] : res.builtOutputs)
|
||||
builtOutputs.insert_or_assign(realisation.id, realisation);
|
||||
worker_proto::write(*store, to, builtOutputs);
|
||||
workerProtoWrite(*store, to, builtOutputs);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
@ -705,7 +709,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
case wopCollectGarbage: {
|
||||
GCOptions options;
|
||||
options.action = (GCOptions::GCAction) readInt(from);
|
||||
options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
|
||||
from >> options.ignoreLiveness >> options.maxFreed;
|
||||
// obsolete fields
|
||||
readInt(from);
|
||||
|
|
@ -775,7 +779,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
else {
|
||||
to << 1
|
||||
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
|
||||
worker_proto::write(*store, to, i->second.references);
|
||||
workerProtoWrite(*store, to, i->second.references);
|
||||
to << i->second.downloadSize
|
||||
<< i->second.narSize;
|
||||
}
|
||||
|
|
@ -786,11 +790,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
SubstitutablePathInfos infos;
|
||||
StorePathCAMap pathsMap = {};
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
|
||||
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
auto paths = WorkerProto<StorePathSet>::read(*store, from);
|
||||
for (auto & path : paths)
|
||||
pathsMap.emplace(path, std::nullopt);
|
||||
} else
|
||||
pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
|
||||
pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
|
||||
logger->startWork();
|
||||
store->querySubstitutablePathInfos(pathsMap, infos);
|
||||
logger->stopWork();
|
||||
|
|
@ -798,7 +802,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
for (auto & i : infos) {
|
||||
to << store->printStorePath(i.first)
|
||||
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
|
||||
worker_proto::write(*store, to, i.second.references);
|
||||
workerProtoWrite(*store, to, i.second.references);
|
||||
to << i.second.downloadSize << i.second.narSize;
|
||||
}
|
||||
break;
|
||||
|
|
@ -808,7 +812,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
logger->startWork();
|
||||
auto paths = store->queryAllValidPaths();
|
||||
logger->stopWork();
|
||||
worker_proto::write(*store, to, paths);
|
||||
workerProtoWrite(*store, to, paths);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -880,7 +884,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
ValidPathInfo info { path, narHash };
|
||||
if (deriver != "")
|
||||
info.deriver = store->parseStorePath(deriver);
|
||||
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
info.references = WorkerProto<StorePathSet>::read(*store, from);
|
||||
from >> info.registrationTime >> info.narSize >> info.ultimate;
|
||||
info.sigs = readStrings<StringSet>(from);
|
||||
info.ca = ContentAddress::parseOpt(readString(from));
|
||||
|
|
@ -931,9 +935,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
uint64_t downloadSize, narSize;
|
||||
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
logger->stopWork();
|
||||
worker_proto::write(*store, to, willBuild);
|
||||
worker_proto::write(*store, to, willSubstitute);
|
||||
worker_proto::write(*store, to, unknown);
|
||||
workerProtoWrite(*store, to, willBuild);
|
||||
workerProtoWrite(*store, to, willSubstitute);
|
||||
workerProtoWrite(*store, to, unknown);
|
||||
to << downloadSize << narSize;
|
||||
break;
|
||||
}
|
||||
|
|
@ -946,7 +950,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
store->registerDrvOutput(Realisation{
|
||||
.id = outputId, .outPath = outputPath});
|
||||
} else {
|
||||
auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
|
||||
auto realisation = WorkerProto<Realisation>::read(*store, from);
|
||||
store->registerDrvOutput(realisation);
|
||||
}
|
||||
logger->stopWork();
|
||||
|
|
@ -961,11 +965,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
|
||||
std::set<StorePath> outPaths;
|
||||
if (info) outPaths.insert(info->outPath);
|
||||
worker_proto::write(*store, to, outPaths);
|
||||
workerProtoWrite(*store, to, outPaths);
|
||||
} else {
|
||||
std::set<Realisation> realisations;
|
||||
if (info) realisations.insert(*info);
|
||||
worker_proto::write(*store, to, realisations);
|
||||
workerProtoWrite(*store, to, realisations);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
@ -1041,7 +1045,7 @@ void processConnection(
|
|||
auto temp = trusted
|
||||
? store->isTrustedClient()
|
||||
: std::optional { NotTrusted };
|
||||
worker_proto::write(*store, to, temp);
|
||||
workerProtoWrite(*store, to, temp);
|
||||
}
|
||||
|
||||
/* Send startup error messages to the client. */
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
#include "derivations.hh"
|
||||
#include "downstream-placeholder.hh"
|
||||
#include "store-api.hh"
|
||||
#include "globals.hh"
|
||||
#include "util.hh"
|
||||
#include "split.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "fs-accessor.hh"
|
||||
#include <boost/container/small_vector.hpp>
|
||||
|
|
@ -35,9 +37,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
|
|||
|
||||
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
||||
{
|
||||
return store.makeFixedOutputPath(
|
||||
return store.makeFixedOutputPathFromCA(
|
||||
outputPathName(drvName, outputName),
|
||||
{ hash, {} });
|
||||
ContentAddressWithReferences::withoutRefs(ca));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -211,29 +213,27 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
|
|||
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store,
|
||||
std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
|
||||
std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
|
||||
{
|
||||
if (hashAlgo != "") {
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
if (hashAlgo.substr(0, 2) == "r:") {
|
||||
method = FileIngestionMethod::Recursive;
|
||||
hashAlgo = hashAlgo.substr(2);
|
||||
}
|
||||
ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
|
||||
if (method == TextIngestionMethod {})
|
||||
experimentalFeatureSettings.require(Xp::DynamicDerivations);
|
||||
const auto hashType = parseHashType(hashAlgo);
|
||||
if (hash == "impure") {
|
||||
if (hashS == "impure") {
|
||||
experimentalFeatureSettings.require(Xp::ImpureDerivations);
|
||||
assert(pathS == "");
|
||||
return DerivationOutput::Impure {
|
||||
.method = std::move(method),
|
||||
.hashType = std::move(hashType),
|
||||
};
|
||||
} else if (hash != "") {
|
||||
} else if (hashS != "") {
|
||||
validatePath(pathS);
|
||||
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
|
||||
return DerivationOutput::CAFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
|
||||
},
|
||||
.ca = ContentAddress::fromParts(
|
||||
std::move(method),
|
||||
std::move(hash)),
|
||||
};
|
||||
} else {
|
||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||
|
|
@ -393,12 +393,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
|
|||
},
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
|
||||
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
|
||||
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
|
||||
s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
|
||||
s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false));
|
||||
},
|
||||
[&](const DerivationOutput::CAFloating & dof) {
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
|
||||
s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
|
||||
s += ','; printUnquotedString(s, "");
|
||||
},
|
||||
[&](const DerivationOutput::Deferred &) {
|
||||
|
|
@ -409,7 +409,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
|
|||
[&](const DerivationOutputImpure & doi) {
|
||||
// FIXME
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
|
||||
s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
|
||||
s += ','; printUnquotedString(s, "impure");
|
||||
}
|
||||
}, i.second.raw());
|
||||
|
|
@ -626,8 +626,8 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
|
|||
for (const auto & i : drv.outputs) {
|
||||
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
|
||||
auto hash = hashString(htSHA256, "fixed:out:"
|
||||
+ dof.hash.printMethodAlgo() + ":"
|
||||
+ dof.hash.hash.to_string(Base16, false) + ":"
|
||||
+ dof.ca.printMethodAlgo() + ":"
|
||||
+ dof.ca.getHash().to_string(Base16, false) + ":"
|
||||
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
||||
outputHashes.insert_or_assign(i.first, std::move(hash));
|
||||
}
|
||||
|
|
@ -749,7 +749,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
|
|||
drv.outputs.emplace(std::move(name), std::move(output));
|
||||
}
|
||||
|
||||
drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
|
||||
drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
|
||||
in >> drv.platform >> drv.builder;
|
||||
drv.args = readStrings<Strings>(in);
|
||||
|
||||
|
|
@ -777,12 +777,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
|
|||
},
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
out << store.printStorePath(dof.path(store, drv.name, i.first))
|
||||
<< dof.hash.printMethodAlgo()
|
||||
<< dof.hash.hash.to_string(Base16, false);
|
||||
<< dof.ca.printMethodAlgo()
|
||||
<< dof.ca.getHash().to_string(Base16, false);
|
||||
},
|
||||
[&](const DerivationOutput::CAFloating & dof) {
|
||||
out << ""
|
||||
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
|
||||
<< (dof.method.renderPrefix() + printHashType(dof.hashType))
|
||||
<< "";
|
||||
},
|
||||
[&](const DerivationOutput::Deferred &) {
|
||||
|
|
@ -792,12 +792,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
|
|||
},
|
||||
[&](const DerivationOutput::Impure & doi) {
|
||||
out << ""
|
||||
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
|
||||
<< (doi.method.renderPrefix() + printHashType(doi.hashType))
|
||||
<< "impure";
|
||||
},
|
||||
}, i.second.raw());
|
||||
}
|
||||
worker_proto::write(store, out, drv.inputSrcs);
|
||||
workerProtoWrite(store, out, drv.inputSrcs);
|
||||
out << drv.platform << drv.builder << drv.args;
|
||||
out << drv.env.size();
|
||||
for (auto & i : drv.env)
|
||||
|
|
@ -811,13 +811,7 @@ std::string hashPlaceholder(const std::string_view outputName)
|
|||
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
|
||||
}
|
||||
|
||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
|
||||
{
|
||||
auto drvNameWithExtension = drvPath.name();
|
||||
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
|
||||
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
|
||||
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
||||
|
|
@ -881,7 +875,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
|
|||
for (auto & outputName : inputOutputs) {
|
||||
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
|
||||
inputRewrites.emplace(
|
||||
downstreamPlaceholder(store, inputDrv, outputName),
|
||||
DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
|
||||
store.printStorePath(*actualPath));
|
||||
resolved.inputSrcs.insert(*actualPath);
|
||||
} else {
|
||||
|
|
@ -942,7 +936,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
|
|||
envHasRightPath(doia.path, i.first);
|
||||
},
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} });
|
||||
auto path = dof.path(store, drvName, i.first);
|
||||
envHasRightPath(path, i.first);
|
||||
},
|
||||
[&](const DerivationOutput::CAFloating &) {
|
||||
|
|
@ -971,15 +965,16 @@ nlohmann::json DerivationOutput::toJSON(
|
|||
},
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
|
||||
res["hashAlgo"] = dof.hash.printMethodAlgo();
|
||||
res["hash"] = dof.hash.hash.to_string(Base16, false);
|
||||
res["hashAlgo"] = dof.ca.printMethodAlgo();
|
||||
res["hash"] = dof.ca.getHash().to_string(Base16, false);
|
||||
// FIXME print refs?
|
||||
},
|
||||
[&](const DerivationOutput::CAFloating & dof) {
|
||||
res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType);
|
||||
res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
|
||||
},
|
||||
[&](const DerivationOutput::Deferred &) {},
|
||||
[&](const DerivationOutput::Impure & doi) {
|
||||
res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType);
|
||||
res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
|
||||
res["impure"] = true;
|
||||
},
|
||||
}, raw());
|
||||
|
|
@ -998,15 +993,15 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
for (const auto & [key, _] : json)
|
||||
keys.insert(key);
|
||||
|
||||
auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> {
|
||||
auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
|
||||
std::string hashAlgo = json["hashAlgo"];
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
if (hashAlgo.substr(0, 2) == "r:") {
|
||||
method = FileIngestionMethod::Recursive;
|
||||
hashAlgo = hashAlgo.substr(2);
|
||||
}
|
||||
auto hashType = parseHashType(hashAlgo);
|
||||
return { method, hashType };
|
||||
// remaining to parse, will be mutated by parsers
|
||||
std::string_view s = hashAlgo;
|
||||
ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
|
||||
if (method == TextIngestionMethod {})
|
||||
xpSettings.require(Xp::DynamicDerivations);
|
||||
auto hashType = parseHashType(s);
|
||||
return { std::move(method), std::move(hashType) };
|
||||
};
|
||||
|
||||
if (keys == (std::set<std::string_view> { "path" })) {
|
||||
|
|
@ -1018,10 +1013,9 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
|
||||
auto [method, hashType] = methodAlgo();
|
||||
auto dof = DerivationOutput::CAFixed {
|
||||
.hash = {
|
||||
.method = method,
|
||||
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
|
||||
},
|
||||
.ca = ContentAddress::fromParts(
|
||||
std::move(method),
|
||||
Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)),
|
||||
};
|
||||
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
|
||||
throw Error("Path doesn't match derivation output");
|
||||
|
|
@ -1032,8 +1026,8 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
xpSettings.require(Xp::CaDerivations);
|
||||
auto [method, hashType] = methodAlgo();
|
||||
return DerivationOutput::CAFloating {
|
||||
.method = method,
|
||||
.hashType = hashType,
|
||||
.method = std::move(method),
|
||||
.hashType = std::move(hashType),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1045,7 +1039,7 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
xpSettings.require(Xp::ImpureDerivations);
|
||||
auto [method, hashType] = methodAlgo();
|
||||
return DerivationOutput::Impure {
|
||||
.method = method,
|
||||
.method = std::move(method),
|
||||
.hashType = hashType,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "hash.hh"
|
||||
#include "content-address.hh"
|
||||
#include "repair-flag.hh"
|
||||
#include "derived-path.hh"
|
||||
#include "sync.hh"
|
||||
#include "comparator.hh"
|
||||
|
||||
|
|
@ -36,9 +37,11 @@ struct DerivationOutputInputAddressed
|
|||
struct DerivationOutputCAFixed
|
||||
{
|
||||
/**
|
||||
* hash used for expected hash computation
|
||||
* Method and hash used for expected hash computation.
|
||||
*
|
||||
* References are not allowed by fiat.
|
||||
*/
|
||||
FixedOutputHash hash;
|
||||
ContentAddress ca;
|
||||
|
||||
/**
|
||||
* Return the \ref StorePath "store path" corresponding to this output
|
||||
|
|
@ -48,7 +51,7 @@ struct DerivationOutputCAFixed
|
|||
*/
|
||||
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
|
||||
|
||||
GENERATE_CMP(DerivationOutputCAFixed, me->hash);
|
||||
GENERATE_CMP(DerivationOutputCAFixed, me->ca);
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -61,7 +64,7 @@ struct DerivationOutputCAFloating
|
|||
/**
|
||||
* How the file system objects will be serialized for hashing
|
||||
*/
|
||||
FileIngestionMethod method;
|
||||
ContentAddressMethod method;
|
||||
|
||||
/**
|
||||
* How the serialization will be hashed
|
||||
|
|
@ -88,7 +91,7 @@ struct DerivationOutputImpure
|
|||
/**
|
||||
* How the file system objects will be serialized for hashing
|
||||
*/
|
||||
FileIngestionMethod method;
|
||||
ContentAddressMethod method;
|
||||
|
||||
/**
|
||||
* How the serialization will be hashed
|
||||
|
|
@ -343,12 +346,14 @@ struct Derivation : BasicDerivation
|
|||
Store & store,
|
||||
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
|
||||
|
||||
/* Check that the derivation is valid and does not present any
|
||||
illegal states.
|
||||
|
||||
This is mainly a matter of checking the outputs, where our C++
|
||||
representation supports all sorts of combinations we do not yet
|
||||
allow. */
|
||||
/**
|
||||
* Check that the derivation is valid and does not present any
|
||||
* illegal states.
|
||||
*
|
||||
* This is mainly a matter of checking the outputs, where our C++
|
||||
* representation supports all sorts of combinations we do not yet
|
||||
* allow.
|
||||
*/
|
||||
void checkInvariants(Store & store, const StorePath & drvPath) const;
|
||||
|
||||
Derivation() = default;
|
||||
|
|
@ -491,17 +496,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
|
|||
*/
|
||||
std::string hashPlaceholder(const std::string_view outputName);
|
||||
|
||||
/**
|
||||
* This creates an opaque and almost certainly unique string
|
||||
* deterministically from a derivation path and output name.
|
||||
*
|
||||
* It is used as a placeholder to allow derivations to refer to
|
||||
* content-addressed paths whose content --- and thus the path
|
||||
* themselves --- isn't yet known. This occurs when a derivation has a
|
||||
* dependency which is a CA derivation.
|
||||
*/
|
||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
|
||||
|
||||
extern const Hash impureOutputHash;
|
||||
|
||||
}
|
||||
|
|
|
|||
39
src/libstore/downstream-placeholder.cc
Normal file
39
src/libstore/downstream-placeholder.cc
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
#include "downstream-placeholder.hh"
|
||||
#include "derivations.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::string DownstreamPlaceholder::render() const
|
||||
{
|
||||
return "/" + hash.to_string(Base32, false);
|
||||
}
|
||||
|
||||
|
||||
DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
|
||||
const StorePath & drvPath,
|
||||
std::string_view outputName)
|
||||
{
|
||||
auto drvNameWithExtension = drvPath.name();
|
||||
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
|
||||
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
|
||||
return DownstreamPlaceholder {
|
||||
hashString(htSHA256, clearText)
|
||||
};
|
||||
}
|
||||
|
||||
DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
|
||||
const DownstreamPlaceholder & placeholder,
|
||||
std::string_view outputName,
|
||||
const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
xpSettings.require(Xp::DynamicDerivations);
|
||||
auto compressed = compressHash(placeholder.hash, 20);
|
||||
auto clearText = "nix-computed-output:"
|
||||
+ compressed.to_string(Base32, false)
|
||||
+ ":" + std::string { outputName };
|
||||
return DownstreamPlaceholder {
|
||||
hashString(htSHA256, clearText)
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
75
src/libstore/downstream-placeholder.hh
Normal file
75
src/libstore/downstream-placeholder.hh
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "hash.hh"
|
||||
#include "path.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* Downstream Placeholders are opaque and almost certainly unique values
|
||||
* used to allow derivations to refer to store objects which are yet to
|
||||
* be built and for we do not yet have store paths for.
|
||||
*
|
||||
* They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`,
|
||||
* except for the cases involving input addressing or fixed outputs
|
||||
* where we do know a store path for the derivation output in advance.
|
||||
*
|
||||
* Unlike `DerivationPath`, however, `DownstreamPlaceholder` is
|
||||
* purposefully opaque and obfuscated. This is so they are hard to
|
||||
* create by accident, and so substituting them (once we know what the
|
||||
* path to store object is) is unlikely to capture other stuff it
|
||||
* shouldn't.
|
||||
*
|
||||
* We use them with `Derivation`: the `render()` method is called to
|
||||
* render an opaque string which can be used in the derivation, and the
|
||||
* resolving logic can substitute those strings for store paths when
|
||||
* resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`.
|
||||
*/
|
||||
class DownstreamPlaceholder
|
||||
{
|
||||
/**
|
||||
* `DownstreamPlaceholder` is just a newtype of `Hash`.
|
||||
* This its only field.
|
||||
*/
|
||||
Hash hash;
|
||||
|
||||
/**
|
||||
* Newtype constructor
|
||||
*/
|
||||
DownstreamPlaceholder(Hash hash) : hash(hash) { }
|
||||
|
||||
public:
|
||||
/**
|
||||
* This creates an opaque and almost certainly unique string
|
||||
* deterministically from the placeholder.
|
||||
*/
|
||||
std::string render() const;
|
||||
|
||||
/**
|
||||
* Create a placeholder for an unknown output of a content-addressed
|
||||
* derivation.
|
||||
*
|
||||
* The derivation itself is known (we have a store path for it), but
|
||||
* the output doesn't yet have a known store path.
|
||||
*/
|
||||
static DownstreamPlaceholder unknownCaOutput(
|
||||
const StorePath & drvPath,
|
||||
std::string_view outputName);
|
||||
|
||||
/**
|
||||
* Create a placehold for the output of an unknown derivation.
|
||||
*
|
||||
* The derivation is not yet known because it is a dynamic
|
||||
* derivaiton --- it is itself an output of another derivation ---
|
||||
* and we just have (another) placeholder for it.
|
||||
*
|
||||
* @param xpSettings Stop-gap to avoid globals during unit tests.
|
||||
*/
|
||||
static DownstreamPlaceholder unknownDerivation(
|
||||
const DownstreamPlaceholder & drvPlaceholder,
|
||||
std::string_view outputName,
|
||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
|
|||
teeSink
|
||||
<< exportMagic
|
||||
<< printStorePath(path);
|
||||
worker_proto::write(*this, teeSink, info->references);
|
||||
workerProtoWrite(*this, teeSink, info->references);
|
||||
teeSink
|
||||
<< (info->deriver ? printStorePath(*info->deriver) : "")
|
||||
<< 0;
|
||||
|
|
@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
|
|||
|
||||
//Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
|
||||
|
||||
auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
|
||||
auto references = WorkerProto<StorePathSet>::read(*this, source);
|
||||
auto deriver = readString(source);
|
||||
auto narHash = hashString(htSHA256, saved.s);
|
||||
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ bool Settings::isWSL1()
|
|||
Path Settings::getDefaultSSLCertFile()
|
||||
{
|
||||
for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
|
||||
if (pathExists(fn)) return fn;
|
||||
if (pathAccessible(fn)) return fn;
|
||||
return "";
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -159,6 +159,15 @@ public:
|
|||
)",
|
||||
{"build-max-jobs"}};
|
||||
|
||||
Setting<unsigned int> maxSubstitutionJobs{
|
||||
this, 16, "max-substitution-jobs",
|
||||
R"(
|
||||
This option defines the maximum number of substitution jobs that Nix
|
||||
will try to run in parallel. The default is `16`. The minimum value
|
||||
one can choose is `1` and lower values will be interpreted as `1`.
|
||||
)",
|
||||
{"substitution-max-jobs"}};
|
||||
|
||||
Setting<unsigned int> buildCores{
|
||||
this,
|
||||
getDefaultCores(),
|
||||
|
|
@ -887,12 +896,11 @@ public:
|
|||
this, {}, "hashed-mirrors",
|
||||
R"(
|
||||
A list of web servers used by `builtins.fetchurl` to obtain files by
|
||||
hash. The default is `http://tarballs.nixos.org/`. Given a hash type
|
||||
*ht* and a base-16 hash *h*, Nix will try to download the file from
|
||||
*hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
|
||||
they have disappeared from their original URI. For example, given
|
||||
the default mirror `http://tarballs.nixos.org/`, when building the
|
||||
derivation
|
||||
hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
|
||||
download the file from *hashed-mirror*/*ht*/*h*. This allows files to
|
||||
be downloaded even if they have disappeared from their original URI.
|
||||
For example, given an example mirror `http://tarballs.nixos.org/`,
|
||||
when building the derivation
|
||||
|
||||
```nix
|
||||
builtins.fetchurl {
|
||||
|
|
@ -972,7 +980,7 @@ public:
|
|||
this, false, "use-xdg-base-directories",
|
||||
R"(
|
||||
If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`.
|
||||
The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md).
|
||||
The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/command-ref/env-common.md).
|
||||
|
||||
[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
|
||||
|
||||
|
|
@ -986,6 +994,18 @@ public:
|
|||
| `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` |
|
||||
| `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` |
|
||||
| `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` |
|
||||
|
||||
If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option.
|
||||
If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`.
|
||||
This can be achieved with the following shell commands:
|
||||
|
||||
```sh
|
||||
nix_state_home=${XDG_STATE_HOME-$HOME/.local/state}/nix
|
||||
mkdir -p $nix_state_home
|
||||
mv $HOME/.nix-profile $nix_state_home/profile
|
||||
mv $HOME/.nix-defexpr $nix_state_home/defexpr
|
||||
mv $HOME/.nix-channels $nix_state_home/channels
|
||||
```
|
||||
)"
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
|||
auto deriver = readString(conn->from);
|
||||
if (deriver != "")
|
||||
info->deriver = parseStorePath(deriver);
|
||||
info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
readLongLong(conn->from); // download size
|
||||
info->narSize = readLongLong(conn->from);
|
||||
|
||||
|
|
@ -180,7 +180,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
|||
<< printStorePath(info.path)
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< info.narHash.to_string(Base16, false);
|
||||
worker_proto::write(*this, conn->to, info.references);
|
||||
workerProtoWrite(*this, conn->to, info.references);
|
||||
conn->to
|
||||
<< info.registrationTime
|
||||
<< info.narSize
|
||||
|
|
@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
|||
conn->to
|
||||
<< exportMagic
|
||||
<< printStorePath(info.path);
|
||||
worker_proto::write(*this, conn->to, info.references);
|
||||
workerProtoWrite(*this, conn->to, info.references);
|
||||
conn->to
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< 0
|
||||
|
|
@ -294,7 +294,7 @@ public:
|
|||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
|
||||
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
|
||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
|
||||
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
|
||||
auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
status.builtOutputs.insert_or_assign(
|
||||
std::move(output.outputName),
|
||||
|
|
@ -344,6 +344,17 @@ public:
|
|||
virtual ref<FSAccessor> getFSAccessor() override
|
||||
{ unsupported("getFSAccessor"); }
|
||||
|
||||
/**
|
||||
* The default instance would schedule the work on the client side, but
|
||||
* for consistency with `buildPaths` and `buildDerivation` it should happen
|
||||
* on the remote side.
|
||||
*
|
||||
* We make this fail for now so we can add implement this properly later
|
||||
* without it being a breaking change.
|
||||
*/
|
||||
void repairPath(const StorePath & path) override
|
||||
{ unsupported("repairPath"); }
|
||||
|
||||
void computeFSClosure(const StorePathSet & paths,
|
||||
StorePathSet & out, bool flipDirection = false,
|
||||
bool includeOutputs = false, bool includeDerivers = false) override
|
||||
|
|
@ -358,10 +369,10 @@ public:
|
|||
conn->to
|
||||
<< cmdQueryClosure
|
||||
<< includeOutputs;
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
workerProtoWrite(*this, conn->to, paths);
|
||||
conn->to.flush();
|
||||
|
||||
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
|
||||
for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
|
||||
out.insert(i);
|
||||
}
|
||||
|
||||
|
|
@ -374,10 +385,10 @@ public:
|
|||
<< cmdQueryValidPaths
|
||||
<< false // lock
|
||||
<< maybeSubstitute;
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
workerProtoWrite(*this, conn->to, paths);
|
||||
conn->to.flush();
|
||||
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
return WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
}
|
||||
|
||||
void connect() override
|
||||
|
|
|
|||
|
|
@ -240,8 +240,6 @@ public:
|
|||
|
||||
void vacuumDB();
|
||||
|
||||
void repairPath(const StorePath & path) override;
|
||||
|
||||
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -57,12 +57,6 @@ $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
|
|||
|
||||
$(d)/build.cc:
|
||||
|
||||
%.gen.hh: %
|
||||
@echo 'R"foo(' >> $@.tmp
|
||||
$(trace-gen) cat $< >> $@.tmp
|
||||
@echo ')foo"' >> $@.tmp
|
||||
@mv $@.tmp $@
|
||||
|
||||
clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
|
||||
|
||||
$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
|
||||
|
|
|
|||
|
|
@ -83,14 +83,15 @@ void Store::computeFSClosure(const StorePath & startPath,
|
|||
}
|
||||
|
||||
|
||||
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
|
||||
const ContentAddress * getDerivationCA(const BasicDerivation & drv)
|
||||
{
|
||||
auto out = drv.outputs.find("out");
|
||||
if (out != drv.outputs.end()) {
|
||||
if (const auto * v = std::get_if<DerivationOutput::CAFixed>(&out->second.raw()))
|
||||
return v->hash;
|
||||
if (out == drv.outputs.end())
|
||||
return nullptr;
|
||||
if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
|
||||
return &dof->ca;
|
||||
}
|
||||
return std::nullopt;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
||||
|
|
@ -140,7 +141,13 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
if (drvState_->lock()->done) return;
|
||||
|
||||
SubstitutablePathInfos infos;
|
||||
querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
|
||||
auto * cap = getDerivationCA(*drv);
|
||||
querySubstitutablePathInfos({
|
||||
{
|
||||
outPath,
|
||||
cap ? std::optional { *cap } : std::nullopt,
|
||||
},
|
||||
}, infos);
|
||||
|
||||
if (infos.empty()) {
|
||||
drvState_->lock()->done = true;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "path-info.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -131,7 +132,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
|
|||
auto narHash = Hash::parseAny(readString(source), htSHA256);
|
||||
ValidPathInfo info(path, narHash);
|
||||
if (deriver != "") info.deriver = store.parseStorePath(deriver);
|
||||
info.references = worker_proto::read(store, source, Phantom<StorePathSet> {});
|
||||
info.references = WorkerProto<StorePathSet>::read(store, source);
|
||||
source >> info.registrationTime >> info.narSize;
|
||||
if (format >= 16) {
|
||||
source >> info.ultimate;
|
||||
|
|
@ -152,7 +153,7 @@ void ValidPathInfo::write(
|
|||
sink << store.printStorePath(path);
|
||||
sink << (deriver ? store.printStorePath(*deriver) : "")
|
||||
<< narHash.to_string(Base16, false);
|
||||
worker_proto::write(store, sink, references);
|
||||
workerProtoWrite(store, sink, references);
|
||||
sink << registrationTime << narSize;
|
||||
if (format >= 16) {
|
||||
sink << ultimate
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ static void checkName(std::string_view path, std::string_view name)
|
|||
if (name.empty())
|
||||
throw BadStorePath("store path '%s' has an empty name", path);
|
||||
if (name.size() > StorePath::MaxPathLen)
|
||||
throw BadStorePath("store path '%s' has a name longer than '%d characters",
|
||||
StorePath::MaxPathLen, path);
|
||||
throw BadStorePath("store path '%s' has a name longer than %d characters",
|
||||
path, StorePath::MaxPathLen);
|
||||
// See nameRegexStr for the definition
|
||||
for (auto c : name)
|
||||
if (!((c >= '0' && c <= '9')
|
||||
|
|
|
|||
|
|
@ -136,6 +136,19 @@ size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
|
|||
return good;
|
||||
}
|
||||
|
||||
|
||||
SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&& outputs)
|
||||
{
|
||||
SingleDrvOutputs ret = std::move(outputs);
|
||||
for (auto it = ret.begin(); it != ret.end(); ) {
|
||||
if (!wanted.contains(it->first))
|
||||
it = ret.erase(it);
|
||||
else
|
||||
++it;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
StorePath RealisedPath::path() const {
|
||||
return std::visit([](auto && arg) { return arg.getPath(); }, raw);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
namespace nix {
|
||||
|
||||
class Store;
|
||||
struct OutputsSpec;
|
||||
|
||||
/**
|
||||
* A general `Realisation` key.
|
||||
|
|
@ -93,6 +94,14 @@ typedef std::map<std::string, Realisation> SingleDrvOutputs;
|
|||
*/
|
||||
typedef std::map<DrvOutput, Realisation> DrvOutputs;
|
||||
|
||||
/**
|
||||
* Filter a SingleDrvOutputs to include only specific output names
|
||||
*
|
||||
* Moves the `outputs` input.
|
||||
*/
|
||||
SingleDrvOutputs filterDrvOutputs(const OutputsSpec&, SingleDrvOutputs&&);
|
||||
|
||||
|
||||
struct OpaquePath {
|
||||
StorePath path;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,189 +18,6 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
namespace worker_proto {
|
||||
|
||||
std::string read(const Store & store, Source & from, Phantom<std::string> _)
|
||||
{
|
||||
return readString(from);
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const std::string & str)
|
||||
{
|
||||
out << str;
|
||||
}
|
||||
|
||||
|
||||
StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
|
||||
{
|
||||
return store.parseStorePath(readString(from));
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const StorePath & storePath)
|
||||
{
|
||||
out << store.printStorePath(storePath);
|
||||
}
|
||||
|
||||
|
||||
std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
|
||||
{
|
||||
auto temp = readNum<uint8_t>(from);
|
||||
switch (temp) {
|
||||
case 0:
|
||||
return std::nullopt;
|
||||
case 1:
|
||||
return { Trusted };
|
||||
case 2:
|
||||
return { NotTrusted };
|
||||
default:
|
||||
throw Error("Invalid trusted status from remote");
|
||||
}
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
|
||||
{
|
||||
if (!optTrusted)
|
||||
out << (uint8_t)0;
|
||||
else {
|
||||
switch (*optTrusted) {
|
||||
case Trusted:
|
||||
out << (uint8_t)1;
|
||||
break;
|
||||
case NotTrusted:
|
||||
out << (uint8_t)2;
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
|
||||
{
|
||||
return ContentAddress::parse(readString(from));
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const ContentAddress & ca)
|
||||
{
|
||||
out << renderContentAddress(ca);
|
||||
}
|
||||
|
||||
|
||||
DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
|
||||
{
|
||||
auto s = readString(from);
|
||||
return DerivedPath::parseLegacy(store, s);
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const DerivedPath & req)
|
||||
{
|
||||
out << req.to_string_legacy(store);
|
||||
}
|
||||
|
||||
|
||||
Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
|
||||
{
|
||||
std::string rawInput = readString(from);
|
||||
return Realisation::fromJSON(
|
||||
nlohmann::json::parse(rawInput),
|
||||
"remote-protocol"
|
||||
);
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const Realisation & realisation)
|
||||
{
|
||||
out << realisation.toJSON().dump();
|
||||
}
|
||||
|
||||
|
||||
DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
|
||||
{
|
||||
return DrvOutput::parse(readString(from));
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
|
||||
{
|
||||
out << drvOutput.to_string();
|
||||
}
|
||||
|
||||
|
||||
KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
|
||||
{
|
||||
auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
|
||||
auto br = worker_proto::read(store, from, Phantom<BuildResult> {});
|
||||
return KeyedBuildResult {
|
||||
std::move(br),
|
||||
/* .path = */ std::move(path),
|
||||
};
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & to, const KeyedBuildResult & res)
|
||||
{
|
||||
worker_proto::write(store, to, res.path);
|
||||
worker_proto::write(store, to, static_cast<const BuildResult &>(res));
|
||||
}
|
||||
|
||||
|
||||
BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
|
||||
{
|
||||
BuildResult res;
|
||||
res.status = (BuildResult::Status) readInt(from);
|
||||
from
|
||||
>> res.errorMsg
|
||||
>> res.timesBuilt
|
||||
>> res.isNonDeterministic
|
||||
>> res.startTime
|
||||
>> res.stopTime;
|
||||
auto builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
res.builtOutputs.insert_or_assign(
|
||||
std::move(output.outputName),
|
||||
std::move(realisation));
|
||||
return res;
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & to, const BuildResult & res)
|
||||
{
|
||||
to
|
||||
<< res.status
|
||||
<< res.errorMsg
|
||||
<< res.timesBuilt
|
||||
<< res.isNonDeterministic
|
||||
<< res.startTime
|
||||
<< res.stopTime;
|
||||
DrvOutputs builtOutputs;
|
||||
for (auto & [output, realisation] : res.builtOutputs)
|
||||
builtOutputs.insert_or_assign(realisation.id, realisation);
|
||||
worker_proto::write(store, to, builtOutputs);
|
||||
}
|
||||
|
||||
|
||||
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
|
||||
{
|
||||
auto s = readString(from);
|
||||
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
|
||||
{
|
||||
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
|
||||
}
|
||||
|
||||
|
||||
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
|
||||
{
|
||||
return ContentAddress::parseOpt(readString(from));
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
|
||||
{
|
||||
out << (caOpt ? renderContentAddress(*caOpt) : "");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
/* TODO: Separate these store impls into different files, give them better names */
|
||||
RemoteStore::RemoteStore(const Params & params)
|
||||
: RemoteStoreConfig(params)
|
||||
|
|
@ -283,7 +100,7 @@ void RemoteStore::initConnection(Connection & conn)
|
|||
}
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
|
||||
conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {});
|
||||
conn.remoteTrustsUs = WorkerProto<std::optional<TrustedFlag>>::read(*this, conn.from);
|
||||
} else {
|
||||
// We don't know the answer; protocol to old.
|
||||
conn.remoteTrustsUs = std::nullopt;
|
||||
|
|
@ -410,12 +227,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
|
|||
return res;
|
||||
} else {
|
||||
conn->to << wopQueryValidPaths;
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
workerProtoWrite(*this, conn->to, paths);
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
|
||||
conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
|
||||
}
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
return WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -425,7 +242,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryAllValidPaths;
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
return WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -442,9 +259,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
|
|||
return res;
|
||||
} else {
|
||||
conn->to << wopQuerySubstitutablePaths;
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
workerProtoWrite(*this, conn->to, paths);
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
return WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -466,7 +283,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
auto deriver = readString(conn->from);
|
||||
if (deriver != "")
|
||||
info.deriver = parseStorePath(deriver);
|
||||
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
info.downloadSize = readLongLong(conn->from);
|
||||
info.narSize = readLongLong(conn->from);
|
||||
infos.insert_or_assign(i.first, std::move(info));
|
||||
|
|
@ -479,9 +296,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
StorePathSet paths;
|
||||
for (auto & path : pathsMap)
|
||||
paths.insert(path.first);
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
workerProtoWrite(*this, conn->to, paths);
|
||||
} else
|
||||
worker_proto::write(*this, conn->to, pathsMap);
|
||||
workerProtoWrite(*this, conn->to, pathsMap);
|
||||
conn.processStderr();
|
||||
size_t count = readNum<size_t>(conn->from);
|
||||
for (size_t n = 0; n < count; n++) {
|
||||
|
|
@ -489,7 +306,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
auto deriver = readString(conn->from);
|
||||
if (deriver != "")
|
||||
info.deriver = parseStorePath(deriver);
|
||||
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
info.downloadSize = readLongLong(conn->from);
|
||||
info.narSize = readLongLong(conn->from);
|
||||
}
|
||||
|
|
@ -532,7 +349,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryReferrers << printStorePath(path);
|
||||
conn.processStderr();
|
||||
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
|
||||
for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
|
||||
referrers.insert(i);
|
||||
}
|
||||
|
||||
|
|
@ -542,7 +359,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryValidDerivers << printStorePath(path);
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
return WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -554,7 +371,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryDerivationOutputs << printStorePath(path);
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
return WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -564,7 +381,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
|
||||
return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
|
||||
} else {
|
||||
// Fallback for old daemon versions.
|
||||
// For floating-CA derivations (and their co-dependencies) this is an
|
||||
|
|
@ -597,6 +414,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
|||
Source & dump,
|
||||
std::string_view name,
|
||||
ContentAddressMethod caMethod,
|
||||
HashType hashType,
|
||||
const StorePathSet & references,
|
||||
RepairFlag repair)
|
||||
{
|
||||
|
|
@ -608,8 +426,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
|||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< caMethod.render();
|
||||
worker_proto::write(*this, conn->to, references);
|
||||
<< caMethod.render(hashType);
|
||||
workerProtoWrite(*this, conn->to, references);
|
||||
conn->to << repair;
|
||||
|
||||
// The dump source may invoke the store, so we need to make some room.
|
||||
|
|
@ -628,26 +446,29 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
|||
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
|
||||
|
||||
std::visit(overloaded {
|
||||
[&](const TextHashMethod & thm) -> void {
|
||||
[&](const TextIngestionMethod & thm) -> void {
|
||||
if (hashType != htSHA256)
|
||||
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
|
||||
name, printHashType(hashType));
|
||||
std::string s = dump.drain();
|
||||
conn->to << wopAddTextToStore << name << s;
|
||||
worker_proto::write(*this, conn->to, references);
|
||||
workerProtoWrite(*this, conn->to, references);
|
||||
conn.processStderr();
|
||||
},
|
||||
[&](const FixedOutputHashMethod & fohm) -> void {
|
||||
[&](const FileIngestionMethod & fim) -> void {
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
|
||||
<< printHashType(fohm.hashType);
|
||||
<< ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (fim == FileIngestionMethod::Recursive ? 1 : 0)
|
||||
<< printHashType(hashType);
|
||||
|
||||
try {
|
||||
conn->to.written = 0;
|
||||
connections->incCapacity();
|
||||
{
|
||||
Finally cleanup([&]() { connections->decCapacity(); });
|
||||
if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
|
||||
if (fim == FileIngestionMethod::Recursive) {
|
||||
dump.drainInto(conn->to);
|
||||
} else {
|
||||
std::string contents = dump.drain();
|
||||
|
|
@ -678,7 +499,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
|||
StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
|
||||
FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
|
||||
{
|
||||
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
|
||||
return addCAToStore(dump, name, method, hashType, references, repair)->path;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -697,7 +518,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
sink
|
||||
<< exportMagic
|
||||
<< printStorePath(info.path);
|
||||
worker_proto::write(*this, sink, info.references);
|
||||
workerProtoWrite(*this, sink, info.references);
|
||||
sink
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< 0 // == no legacy signature
|
||||
|
|
@ -707,7 +528,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
|
||||
conn.processStderr(0, source2.get());
|
||||
|
||||
auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
assert(importedPaths.size() <= 1);
|
||||
}
|
||||
|
||||
|
|
@ -716,7 +537,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
<< printStorePath(info.path)
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< info.narHash.to_string(Base16, false);
|
||||
worker_proto::write(*this, conn->to, info.references);
|
||||
workerProtoWrite(*this, conn->to, info.references);
|
||||
conn->to << info.registrationTime << info.narSize
|
||||
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
|
||||
<< repair << !checkSigs;
|
||||
|
|
@ -778,7 +599,7 @@ StorePath RemoteStore::addTextToStore(
|
|||
RepairFlag repair)
|
||||
{
|
||||
StringSource source(s);
|
||||
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
|
||||
return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
|
||||
}
|
||||
|
||||
void RemoteStore::registerDrvOutput(const Realisation & info)
|
||||
|
|
@ -789,7 +610,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
|
|||
conn->to << info.id.to_string();
|
||||
conn->to << std::string(info.outPath.to_string());
|
||||
} else {
|
||||
worker_proto::write(*this, conn->to, info);
|
||||
workerProtoWrite(*this, conn->to, info);
|
||||
}
|
||||
conn.processStderr();
|
||||
}
|
||||
|
|
@ -811,14 +632,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
|
|||
|
||||
auto real = [&]() -> std::shared_ptr<const Realisation> {
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
|
||||
auto outPaths = worker_proto::read(
|
||||
*this, conn->from, Phantom<std::set<StorePath>> {});
|
||||
auto outPaths = WorkerProto<std::set<StorePath>>::read(
|
||||
*this, conn->from);
|
||||
if (outPaths.empty())
|
||||
return nullptr;
|
||||
return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
|
||||
} else {
|
||||
auto realisations = worker_proto::read(
|
||||
*this, conn->from, Phantom<std::set<Realisation>> {});
|
||||
auto realisations = WorkerProto<std::set<Realisation>>::read(
|
||||
*this, conn->from);
|
||||
if (realisations.empty())
|
||||
return nullptr;
|
||||
return std::make_shared<const Realisation>(*realisations.begin());
|
||||
|
|
@ -832,7 +653,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
|
|||
static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
|
||||
{
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
|
||||
worker_proto::write(store, conn->to, reqs);
|
||||
workerProtoWrite(store, conn->to, reqs);
|
||||
} else {
|
||||
Strings ss;
|
||||
for (auto & p : reqs) {
|
||||
|
|
@ -902,7 +723,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
|
|||
writeDerivedPaths(*this, conn, paths);
|
||||
conn->to << buildMode;
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<std::vector<KeyedBuildResult>> {});
|
||||
return WorkerProto<std::vector<KeyedBuildResult>>::read(*this, conn->from);
|
||||
} else {
|
||||
// Avoid deadlock.
|
||||
conn_.reset();
|
||||
|
|
@ -985,7 +806,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
|
|||
conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
|
||||
}
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
|
||||
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
|
||||
auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
res.builtOutputs.insert_or_assign(
|
||||
std::move(output.outputName),
|
||||
|
|
@ -1044,7 +865,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
|
||||
conn->to
|
||||
<< wopCollectGarbage << options.action;
|
||||
worker_proto::write(*this, conn->to, options.pathsToDelete);
|
||||
workerProtoWrite(*this, conn->to, options.pathsToDelete);
|
||||
conn->to << options.ignoreLiveness
|
||||
<< options.maxFreed
|
||||
/* removed options */
|
||||
|
|
@ -1103,9 +924,9 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
conn->to << wopQueryMissing;
|
||||
writeDerivedPaths(*this, conn, targets);
|
||||
conn.processStderr();
|
||||
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
|
||||
conn->from >> downloadSize >> narSize;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ public:
|
|||
Source & dump,
|
||||
std::string_view name,
|
||||
ContentAddressMethod caMethod,
|
||||
HashType hashType,
|
||||
const StorePathSet & references,
|
||||
RepairFlag repair);
|
||||
|
||||
|
|
@ -136,6 +137,17 @@ public:
|
|||
|
||||
bool verifyStore(bool checkContents, RepairFlag repair) override;
|
||||
|
||||
/**
|
||||
* The default instance would schedule the work on the client side, but
|
||||
* for consistency with `buildPaths` and `buildDerivation` it should happen
|
||||
* on the remote side.
|
||||
*
|
||||
* We make this fail for now so we can add implement this properly later
|
||||
* without it being a breaking change.
|
||||
*/
|
||||
void repairPath(const StorePath & path) override
|
||||
{ unsupported("repairPath"); }
|
||||
|
||||
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
|
||||
|
||||
void queryMissing(const std::vector<DerivedPath> & targets,
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
|
|||
args.push_back("-oLocalCommand=echo started");
|
||||
}
|
||||
|
||||
bool SSHMaster::isMasterRunning() {
|
||||
auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true});
|
||||
return res.first == 0;
|
||||
}
|
||||
|
||||
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
|
||||
{
|
||||
Path socketPath = startMaster();
|
||||
|
|
@ -97,7 +102,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
|
|||
|
||||
// Wait for the SSH connection to be established,
|
||||
// So that we don't overwrite the password prompt with our progress bar.
|
||||
if (!fakeSSH && !useMaster) {
|
||||
if (!fakeSSH && !useMaster && !isMasterRunning()) {
|
||||
std::string reply;
|
||||
try {
|
||||
reply = readLine(out.readSide.get());
|
||||
|
|
@ -133,6 +138,8 @@ Path SSHMaster::startMaster()
|
|||
logger->pause();
|
||||
Finally cleanup = [&]() { logger->resume(); };
|
||||
|
||||
bool wasMasterRunning = isMasterRunning();
|
||||
|
||||
state->sshMaster = startProcess([&]() {
|
||||
restoreProcessContext();
|
||||
|
||||
|
|
@ -152,13 +159,15 @@ Path SSHMaster::startMaster()
|
|||
|
||||
out.writeSide = -1;
|
||||
|
||||
std::string reply;
|
||||
try {
|
||||
reply = readLine(out.readSide.get());
|
||||
} catch (EndOfFile & e) { }
|
||||
if (!wasMasterRunning) {
|
||||
std::string reply;
|
||||
try {
|
||||
reply = readLine(out.readSide.get());
|
||||
} catch (EndOfFile & e) { }
|
||||
|
||||
if (reply != "started")
|
||||
throw Error("failed to start SSH master connection to '%s'", host);
|
||||
if (reply != "started")
|
||||
throw Error("failed to start SSH master connection to '%s'", host);
|
||||
}
|
||||
|
||||
return state->socketPath;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ private:
|
|||
Sync<State> state_;
|
||||
|
||||
void addCommonSSHOpts(Strings & args);
|
||||
bool isMasterRunning();
|
||||
|
||||
public:
|
||||
|
||||
|
|
|
|||
|
|
@ -679,8 +679,7 @@ public:
|
|||
* Repair the contents of the given path by redownloading it using
|
||||
* a substituter (if available).
|
||||
*/
|
||||
virtual void repairPath(const StorePath & path)
|
||||
{ unsupported("repairPath"); }
|
||||
virtual void repairPath(const StorePath & path);
|
||||
|
||||
/**
|
||||
* Add signatures to the specified store path. The signatures are
|
||||
|
|
@ -1022,7 +1021,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
|
|||
*/
|
||||
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
|
||||
|
||||
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
|
||||
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
|
||||
|
||||
std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||
Store & store,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,14 @@ class CaDerivationTest : public DerivationTest
|
|||
}
|
||||
};
|
||||
|
||||
class DynDerivationTest : public DerivationTest
|
||||
{
|
||||
void SetUp() override
|
||||
{
|
||||
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
|
||||
}
|
||||
};
|
||||
|
||||
class ImpureDerivationTest : public DerivationTest
|
||||
{
|
||||
void SetUp() override
|
||||
|
|
@ -66,20 +74,47 @@ TEST_JSON(DerivationTest, inputAddressed,
|
|||
}),
|
||||
"drv-name", "output-name")
|
||||
|
||||
TEST_JSON(DerivationTest, caFixed,
|
||||
TEST_JSON(DerivationTest, caFixedFlat,
|
||||
R"({
|
||||
"hashAlgo": "sha256",
|
||||
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
|
||||
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
|
||||
})",
|
||||
(DerivationOutput::CAFixed {
|
||||
.ca = FixedOutputHash {
|
||||
.method = FileIngestionMethod::Flat,
|
||||
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
|
||||
},
|
||||
}),
|
||||
"drv-name", "output-name")
|
||||
|
||||
TEST_JSON(DerivationTest, caFixedNAR,
|
||||
R"({
|
||||
"hashAlgo": "r:sha256",
|
||||
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
|
||||
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
|
||||
})",
|
||||
(DerivationOutput::CAFixed {
|
||||
.hash = {
|
||||
.ca = FixedOutputHash {
|
||||
.method = FileIngestionMethod::Recursive,
|
||||
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
|
||||
},
|
||||
}),
|
||||
"drv-name", "output-name")
|
||||
|
||||
TEST_JSON(DynDerivationTest, caFixedText,
|
||||
R"({
|
||||
"hashAlgo": "text:sha256",
|
||||
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
|
||||
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
|
||||
})",
|
||||
(DerivationOutput::CAFixed {
|
||||
.ca = TextHash {
|
||||
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
|
||||
},
|
||||
}),
|
||||
"drv-name", "output-name")
|
||||
|
||||
TEST_JSON(CaDerivationTest, caFloating,
|
||||
R"({
|
||||
"hashAlgo": "r:sha256"
|
||||
|
|
|
|||
|
|
@ -27,11 +27,13 @@ Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
|
|||
|
||||
Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
|
||||
{
|
||||
switch (*gen::inRange<uint8_t>(0, 1)) {
|
||||
switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
|
||||
case 0:
|
||||
return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
|
||||
default:
|
||||
case 1:
|
||||
return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
|
||||
default:
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
33
src/libstore/tests/downstream-placeholder.cc
Normal file
33
src/libstore/tests/downstream-placeholder.cc
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#include <gtest/gtest.h>
|
||||
|
||||
#include "downstream-placeholder.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
TEST(DownstreamPlaceholder, unknownCaOutput) {
|
||||
ASSERT_EQ(
|
||||
DownstreamPlaceholder::unknownCaOutput(
|
||||
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
|
||||
"out").render(),
|
||||
"/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz");
|
||||
}
|
||||
|
||||
TEST(DownstreamPlaceholder, unknownDerivation) {
|
||||
/**
|
||||
* We set these in tests rather than the regular globals so we don't have
|
||||
* to worry about race conditions if the tests run concurrently.
|
||||
*/
|
||||
ExperimentalFeatureSettings mockXpSettings;
|
||||
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
|
||||
|
||||
ASSERT_EQ(
|
||||
DownstreamPlaceholder::unknownDerivation(
|
||||
DownstreamPlaceholder::unknownCaOutput(
|
||||
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" },
|
||||
"out"),
|
||||
"out",
|
||||
mockXpSettings).render(),
|
||||
"/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -206,15 +206,17 @@ using namespace nix;
|
|||
|
||||
Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
|
||||
{
|
||||
switch (*gen::inRange<uint8_t>(0, 1)) {
|
||||
switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
|
||||
case 0:
|
||||
return gen::just((OutputsSpec) OutputsSpec::All { });
|
||||
default:
|
||||
case 1:
|
||||
return gen::just((OutputsSpec) OutputsSpec::Names {
|
||||
*gen::nonEmpty(gen::container<StringSet>(gen::map(
|
||||
gen::arbitrary<StorePathName>(),
|
||||
[](StorePathName n) { return n.name; }))),
|
||||
});
|
||||
default:
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
192
src/libstore/worker-protocol.cc
Normal file
192
src/libstore/worker-protocol.cc
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
#include "serialise.hh"
|
||||
#include "util.hh"
|
||||
#include "path-with-outputs.hh"
|
||||
#include "store-api.hh"
|
||||
#include "build-result.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "archive.hh"
|
||||
#include "derivations.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::string WorkerProto<std::string>::read(const Store & store, Source & from)
|
||||
{
|
||||
return readString(from);
|
||||
}
|
||||
|
||||
void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
|
||||
{
|
||||
out << str;
|
||||
}
|
||||
|
||||
|
||||
StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
|
||||
{
|
||||
return store.parseStorePath(readString(from));
|
||||
}
|
||||
|
||||
void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
|
||||
{
|
||||
out << store.printStorePath(storePath);
|
||||
}
|
||||
|
||||
|
||||
std::optional<TrustedFlag> WorkerProto<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
|
||||
{
|
||||
auto temp = readNum<uint8_t>(from);
|
||||
switch (temp) {
|
||||
case 0:
|
||||
return std::nullopt;
|
||||
case 1:
|
||||
return { Trusted };
|
||||
case 2:
|
||||
return { NotTrusted };
|
||||
default:
|
||||
throw Error("Invalid trusted status from remote");
|
||||
}
|
||||
}
|
||||
|
||||
void WorkerProto<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
|
||||
{
|
||||
if (!optTrusted)
|
||||
out << (uint8_t)0;
|
||||
else {
|
||||
switch (*optTrusted) {
|
||||
case Trusted:
|
||||
out << (uint8_t)1;
|
||||
break;
|
||||
case NotTrusted:
|
||||
out << (uint8_t)2;
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
|
||||
{
|
||||
return ContentAddress::parse(readString(from));
|
||||
}
|
||||
|
||||
void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
|
||||
{
|
||||
out << renderContentAddress(ca);
|
||||
}
|
||||
|
||||
|
||||
DerivedPath WorkerProto<DerivedPath>::read(const Store & store, Source & from)
|
||||
{
|
||||
auto s = readString(from);
|
||||
return DerivedPath::parseLegacy(store, s);
|
||||
}
|
||||
|
||||
void WorkerProto<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
|
||||
{
|
||||
out << req.to_string_legacy(store);
|
||||
}
|
||||
|
||||
|
||||
Realisation WorkerProto<Realisation>::read(const Store & store, Source & from)
|
||||
{
|
||||
std::string rawInput = readString(from);
|
||||
return Realisation::fromJSON(
|
||||
nlohmann::json::parse(rawInput),
|
||||
"remote-protocol"
|
||||
);
|
||||
}
|
||||
|
||||
void WorkerProto<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
|
||||
{
|
||||
out << realisation.toJSON().dump();
|
||||
}
|
||||
|
||||
|
||||
DrvOutput WorkerProto<DrvOutput>::read(const Store & store, Source & from)
|
||||
{
|
||||
return DrvOutput::parse(readString(from));
|
||||
}
|
||||
|
||||
void WorkerProto<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
|
||||
{
|
||||
out << drvOutput.to_string();
|
||||
}
|
||||
|
||||
|
||||
KeyedBuildResult WorkerProto<KeyedBuildResult>::read(const Store & store, Source & from)
|
||||
{
|
||||
auto path = WorkerProto<DerivedPath>::read(store, from);
|
||||
auto br = WorkerProto<BuildResult>::read(store, from);
|
||||
return KeyedBuildResult {
|
||||
std::move(br),
|
||||
/* .path = */ std::move(path),
|
||||
};
|
||||
}
|
||||
|
||||
void WorkerProto<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
|
||||
{
|
||||
workerProtoWrite(store, to, res.path);
|
||||
workerProtoWrite(store, to, static_cast<const BuildResult &>(res));
|
||||
}
|
||||
|
||||
|
||||
BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
|
||||
{
|
||||
BuildResult res;
|
||||
res.status = (BuildResult::Status) readInt(from);
|
||||
from
|
||||
>> res.errorMsg
|
||||
>> res.timesBuilt
|
||||
>> res.isNonDeterministic
|
||||
>> res.startTime
|
||||
>> res.stopTime;
|
||||
auto builtOutputs = WorkerProto<DrvOutputs>::read(store, from);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
res.builtOutputs.insert_or_assign(
|
||||
std::move(output.outputName),
|
||||
std::move(realisation));
|
||||
return res;
|
||||
}
|
||||
|
||||
void WorkerProto<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
|
||||
{
|
||||
to
|
||||
<< res.status
|
||||
<< res.errorMsg
|
||||
<< res.timesBuilt
|
||||
<< res.isNonDeterministic
|
||||
<< res.startTime
|
||||
<< res.stopTime;
|
||||
DrvOutputs builtOutputs;
|
||||
for (auto & [output, realisation] : res.builtOutputs)
|
||||
builtOutputs.insert_or_assign(realisation.id, realisation);
|
||||
workerProtoWrite(store, to, builtOutputs);
|
||||
}
|
||||
|
||||
|
||||
std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
|
||||
{
|
||||
auto s = readString(from);
|
||||
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
|
||||
}
|
||||
|
||||
void WorkerProto<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
|
||||
{
|
||||
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
|
||||
}
|
||||
|
||||
|
||||
std::optional<ContentAddress> WorkerProto<std::optional<ContentAddress>>::read(const Store & store, Source & from)
|
||||
{
|
||||
return ContentAddress::parseOpt(readString(from));
|
||||
}
|
||||
|
||||
void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
|
||||
{
|
||||
out << (caOpt ? renderContentAddress(*caOpt) : "");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "store-api.hh"
|
||||
#include "serialise.hh"
|
||||
|
||||
namespace nix {
|
||||
|
|
@ -79,41 +78,81 @@ typedef enum {
|
|||
class Store;
|
||||
struct Source;
|
||||
|
||||
// items being serialized
|
||||
struct DerivedPath;
|
||||
struct DrvOutput;
|
||||
struct Realisation;
|
||||
struct BuildResult;
|
||||
struct KeyedBuildResult;
|
||||
enum TrustedFlag : bool;
|
||||
|
||||
|
||||
/**
|
||||
* Used to guide overloading
|
||||
* Data type for canonical pairs of serializers for the worker protocol.
|
||||
*
|
||||
* See https://en.cppreference.com/w/cpp/language/adl for the broader
|
||||
* concept of what is going on here.
|
||||
*/
|
||||
template<typename T>
|
||||
struct Phantom {};
|
||||
struct WorkerProto {
|
||||
static T read(const Store & store, Source & from);
|
||||
static void write(const Store & store, Sink & out, const T & t);
|
||||
};
|
||||
|
||||
/**
|
||||
* Wrapper function around `WorkerProto<T>::write` that allows us to
|
||||
* infer the type instead of having to write it down explicitly.
|
||||
*/
|
||||
template<typename T>
|
||||
void workerProtoWrite(const Store & store, Sink & out, const T & t)
|
||||
{
|
||||
WorkerProto<T>::write(store, out, t);
|
||||
}
|
||||
|
||||
namespace worker_proto {
|
||||
/* FIXME maybe move more stuff inside here */
|
||||
/**
|
||||
* Declare a canonical serializer pair for the worker protocol.
|
||||
*
|
||||
* We specialize the struct merely to indicate that we are implementing
|
||||
* the function for the given type.
|
||||
*
|
||||
* Some sort of `template<...>` must be used with the caller for this to
|
||||
* be legal specialization syntax. See below for what that looks like in
|
||||
* practice.
|
||||
*/
|
||||
#define MAKE_WORKER_PROTO(T) \
|
||||
struct WorkerProto< T > { \
|
||||
static T read(const Store & store, Source & from); \
|
||||
static void write(const Store & store, Sink & out, const T & t); \
|
||||
};
|
||||
|
||||
#define MAKE_WORKER_PROTO(TEMPLATE, T) \
|
||||
TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
|
||||
TEMPLATE void write(const Store & store, Sink & out, const T & str)
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::string);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(StorePath);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(ContentAddress);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(DerivedPath);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(Realisation);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(DrvOutput);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(BuildResult);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(KeyedBuildResult);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::optional<TrustedFlag>);
|
||||
|
||||
MAKE_WORKER_PROTO(, std::string);
|
||||
MAKE_WORKER_PROTO(, StorePath);
|
||||
MAKE_WORKER_PROTO(, ContentAddress);
|
||||
MAKE_WORKER_PROTO(, DerivedPath);
|
||||
MAKE_WORKER_PROTO(, Realisation);
|
||||
MAKE_WORKER_PROTO(, DrvOutput);
|
||||
MAKE_WORKER_PROTO(, BuildResult);
|
||||
MAKE_WORKER_PROTO(, KeyedBuildResult);
|
||||
MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
|
||||
template<typename T>
|
||||
MAKE_WORKER_PROTO(std::vector<T>);
|
||||
template<typename T>
|
||||
MAKE_WORKER_PROTO(std::set<T>);
|
||||
|
||||
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
|
||||
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
|
||||
|
||||
#define X_ template<typename K, typename V>
|
||||
#define Y_ std::map<K, V>
|
||||
MAKE_WORKER_PROTO(X_, Y_);
|
||||
template<typename K, typename V>
|
||||
#define X_ std::map<K, V>
|
||||
MAKE_WORKER_PROTO(X_);
|
||||
#undef X_
|
||||
#undef Y_
|
||||
|
||||
/**
|
||||
* These use the empty string for the null case, relying on the fact
|
||||
|
|
@ -129,72 +168,72 @@ MAKE_WORKER_PROTO(X_, Y_);
|
|||
* worker protocol harder to implement in other languages where such
|
||||
* specializations may not be allowed.
|
||||
*/
|
||||
MAKE_WORKER_PROTO(, std::optional<StorePath>);
|
||||
MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::optional<StorePath>);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::optional<ContentAddress>);
|
||||
|
||||
template<typename T>
|
||||
std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
|
||||
std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
|
||||
{
|
||||
std::vector<T> resSet;
|
||||
auto size = readNum<size_t>(from);
|
||||
while (size--) {
|
||||
resSet.push_back(read(store, from, Phantom<T> {}));
|
||||
resSet.push_back(WorkerProto<T>::read(store, from));
|
||||
}
|
||||
return resSet;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void write(const Store & store, Sink & out, const std::vector<T> & resSet)
|
||||
void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
|
||||
{
|
||||
out << resSet.size();
|
||||
for (auto & key : resSet) {
|
||||
write(store, out, key);
|
||||
WorkerProto<T>::write(store, out, key);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
|
||||
std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
|
||||
{
|
||||
std::set<T> resSet;
|
||||
auto size = readNum<size_t>(from);
|
||||
while (size--) {
|
||||
resSet.insert(read(store, from, Phantom<T> {}));
|
||||
resSet.insert(WorkerProto<T>::read(store, from));
|
||||
}
|
||||
return resSet;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void write(const Store & store, Sink & out, const std::set<T> & resSet)
|
||||
void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
|
||||
{
|
||||
out << resSet.size();
|
||||
for (auto & key : resSet) {
|
||||
write(store, out, key);
|
||||
WorkerProto<T>::write(store, out, key);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename K, typename V>
|
||||
std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
|
||||
std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
|
||||
{
|
||||
std::map<K, V> resMap;
|
||||
auto size = readNum<size_t>(from);
|
||||
while (size--) {
|
||||
auto k = read(store, from, Phantom<K> {});
|
||||
auto v = read(store, from, Phantom<V> {});
|
||||
auto k = WorkerProto<K>::read(store, from);
|
||||
auto v = WorkerProto<V>::read(store, from);
|
||||
resMap.insert_or_assign(std::move(k), std::move(v));
|
||||
}
|
||||
return resMap;
|
||||
}
|
||||
|
||||
template<typename K, typename V>
|
||||
void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
|
||||
void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
|
||||
{
|
||||
out << resMap.size();
|
||||
for (auto & i : resMap) {
|
||||
write(store, out, i.first);
|
||||
write(store, out, i.second);
|
||||
WorkerProto<K>::write(store, out, i.first);
|
||||
WorkerProto<V>::write(store, out, i.second);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue