nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-19 16:59:35 +01:00
Code Activity

Merge commit 'd5cef6c33a' into ca-drv-exotic

Browse source
This commit is contained in:
John Ericson 2021-04-05 19:06:37 -04:00
commit 386765e3ff
35 changed files with 502 additions and 216 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/libstore/build/derivation-goal.cc
View file

@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
state = &DerivationGoal::getDerivation;
name = fmt(
"building of '%s' from .drv file",
StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
state = &DerivationGoal::haveDerivation;
name = fmt(
"building of '%s' from in-memory derivation",
StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);

16
src/libstore/build/entry-points.cc
View file

@ -6,16 +6,20 @@
namespace nix {
void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
void Store::buildPaths(const std::vector<BuildableReq> & reqs, BuildMode buildMode)
{
Worker worker(*this);
Goals goals;
for (auto & path : drvPaths) {
if (path.path.isDerivation())
goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
else
goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
for (auto & br : reqs) {
std::visit(overloaded {
[&](BuildableReqFromDrv bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](BuildableOpaque bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
}
worker.run(goals);

52
src/libstore/build/local-derivation-goal.cc
View file

@ -1190,6 +1190,26 @@ void LocalDerivationGoal::writeStructuredAttrs()
chownToBuilder(tmpDir + "/.attrs.sh");
}
static StorePath pathPartOfReq(const BuildableReq & req)
{
return std::visit(overloaded {
[&](BuildableOpaque bo) {
return bo.path;
},
[&](BuildableReqFromDrv bfd) {
return bfd.drvPath;
},
}, req.raw());
}
bool LocalDerivationGoal::isAllowed(const BuildableReq & req)
{
return this->isAllowed(pathPartOfReq(req));
}
struct RestrictedStoreConfig : virtual LocalFSStoreConfig
{
using LocalFSStoreConfig::LocalFSStoreConfig;
@ -1312,25 +1332,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
// an allowed derivation
{ throw Error("queryRealisation"); }
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override
{
if (buildMode != bmNormal) throw Error("unsupported build mode");
StorePathSet newPaths;
for (auto & path : paths) {
if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
for (auto & req : paths) {
if (!goal.isAllowed(req))
throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
}
next->buildPaths(paths, buildMode);
for (auto & path : paths) {
if (!path.path.isDerivation()) continue;
auto outputs = next->queryDerivationOutputMap(path.path);
for (auto & output : outputs)
if (wantOutput(output.first, path.outputs))
newPaths.insert(output.second);
auto p = std::get_if<BuildableReqFromDrv>(&path);
if (!p) continue;
auto & bfd = *p;
auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs))
newPaths.insert(outputPath);
}
StorePathSet closure;
@ -1358,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
void addSignatures(const StorePath & storePath, const StringSet & sigs) override
{ unsupported("addSignatures"); }
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
void queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override
{
@ -1366,12 +1388,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
client about what paths will be built/substituted or are
already present. Probably not a big deal. */
std::vector<StorePathWithOutputs> allowed;
for (auto & path : targets) {
if (goal.isAllowed(path.path))
allowed.emplace_back(path);
std::vector<BuildableReq> allowed;
for (auto & req : targets) {
if (goal.isAllowed(req))
allowed.emplace_back(req);
else
unknown.insert(path.path);
unknown.insert(pathPartOfReq(req));
}
next->queryMissing(allowed, willBuild, willSubstitute,

1
src/libstore/build/local-derivation-goal.hh
View file

@ -116,6 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal
{
return inputPaths.count(path) || addedPaths.count(path);
}
bool isAllowed(const BuildableReq & req);
friend struct RestrictedStore;

6
src/libstore/build/worker.cc
View file

@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal)
void Worker::run(const Goals & _topGoals)
{
std::vector<nix::StorePathWithOutputs> topPaths;
std::vector<nix::BuildableReq> topPaths;
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back({goal->drvPath, goal->wantedOutputs});
topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs});
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back({goal->storePath});
topPaths.push_back(BuildableOpaque{goal->storePath});
}
}

77
src/libstore/buildable.cc Normal file
View file

@ -0,0 +1,77 @@
#include "buildable.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix {
nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
nlohmann::json res;
res["path"] = store->printStorePath(path);
return res;
}
nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = path ? store->printStorePath(*path) : "";
}
return res;
}
nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
auto res = nlohmann::json::array();
for (const Buildable & buildable : buildables) {
std::visit([&res, store](const auto & buildable) {
res.push_back(buildable.toJSON(store));
}, buildable);
}
return res;
}
std::string BuildableOpaque::to_string(const Store & store) const {
return store.printStorePath(path);
}
std::string BuildableReqFromDrv::to_string(const Store & store) const {
return store.printStorePath(drvPath)
+ "!"
+ (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
}
std::string BuildableReq::to_string(const Store & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
this->raw());
}
BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s)
{
return {store.parseStorePath(s)};
}
BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s)
{
size_t n = s.find("!");
assert(n != s.npos);
auto drvPath = store.parseStorePath(s.substr(0, n));
auto outputsS = s.substr(n + 1);
std::set<string> outputs;
if (outputsS != "*")
outputs = tokenizeString<std::set<string>>(outputsS);
return {drvPath, outputs};
}
BuildableReq BuildableReq::parse(const Store & store, std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? (BuildableReq) BuildableOpaque::parse(store, s)
: (BuildableReq) BuildableReqFromDrv::parse(store, s);
}
}

65
src/libstore/buildable.hh Normal file
View file

@ -0,0 +1,65 @@
#pragma once
#include "util.hh"
#include "path.hh"
#include <optional>
#include <variant>
#include <nlohmann/json_fwd.hpp>
namespace nix {
class Store;
struct BuildableOpaque {
StorePath path;
nlohmann::json toJSON(ref<Store> store) const;
std::string to_string(const Store & store) const;
static BuildableOpaque parse(const Store & store, std::string_view);
};
struct BuildableReqFromDrv {
StorePath drvPath;
std::set<std::string> outputs;
std::string to_string(const Store & store) const;
static BuildableReqFromDrv parse(const Store & store, std::string_view);
};
using _BuildableReqRaw = std::variant<
BuildableOpaque,
BuildableReqFromDrv
>;
struct BuildableReq : _BuildableReqRaw {
using Raw = _BuildableReqRaw;
using Raw::Raw;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
std::string to_string(const Store & store) const;
static BuildableReq parse(const Store & store, std::string_view);
};
struct BuildableFromDrv {
StorePath drvPath;
std::map<std::string, std::optional<StorePath>> outputs;
nlohmann::json toJSON(ref<Store> store) const;
static BuildableFromDrv parse(const Store & store, std::string_view);
};
using Buildable = std::variant<
BuildableOpaque,
BuildableFromDrv
>;
typedef std::vector<Buildable> Buildables;
nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
}

21
src/libstore/daemon.cc
View file

@ -2,6 +2,7 @@
#include "monitor-fd.hh"
#include "worker-protocol.hh"
#include "store-api.hh"
#include "path-with-outputs.hh"
#include "finally.hh"
#include "affinity.hh"
#include "archive.hh"
@ -259,6 +260,18 @@ static void writeValidPathInfo(
}
}
static std::vector<BuildableReq> readBuildableReqs(Store & store, unsigned int clientVersion, Source & from)
{
std::vector<BuildableReq> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
reqs = worker_proto::read(store, from, Phantom<std::vector<BuildableReq>> {});
} else {
for (auto & s : readStrings<Strings>(from))
reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq());
}
return reqs;
}
static void performOp(TunnelLogger * logger, ref<Store> store,
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
Source & from, BufferedSink & to, unsigned int op)
@ -497,9 +510,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopBuildPaths: {
std::vector<StorePathWithOutputs> drvs;
for (auto & s : readStrings<Strings>(from))
drvs.push_back(store->parsePathWithOutputs(s));
auto drvs = readBuildableReqs(*store, clientVersion, from);
BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from);
@ -863,9 +874,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryMissing: {
std::vector<StorePathWithOutputs> targets;
for (auto & s : readStrings<Strings>(from))
targets.push_back(store->parsePathWithOutputs(s));
auto targets = readBuildableReqs(*store, clientVersion, from);
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;

8
src/libstore/derivations.cc
View file

@ -588,14 +588,6 @@ std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& d
}
std::string StorePathWithOutputs::to_string(const Store & store) const
{
return outputs.empty()
? store.printStorePath(path)
: store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
}
bool wantOutput(const string & output, const std::set<string> & wanted)
{
return wanted.empty() || wanted.find(output) != wanted.end();

16
src/libstore/legacy-ssh-store.cc
View file

@ -3,6 +3,7 @@
#include "remote-store.hh"
#include "serve-protocol.hh"
#include "store-api.hh"
#include "path-with-outputs.hh"
#include "worker-protocol.hh"
#include "ssh.hh"
#include "derivations.hh"
@ -265,14 +266,23 @@ public:
return status;
}
void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override
void buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode) override
{
auto conn(connections->get());
conn->to << cmdBuildPaths;
Strings ss;
for (auto & p : drvPaths)
ss.push_back(p.to_string(*this));
for (auto & p : drvPaths) {
auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
std::visit(overloaded {
[&](StorePathWithOutputs s) {
ss.push_back(s.to_string(*this));
},
[&](StorePath drvPath) {
throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
},
}, sOrDrvPath);
}
conn->to << ss;
putBuildSettings(*conn);

49
src/libstore/misc.cc
View file

@ -128,7 +128,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
return std::nullopt;
}
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
void Store::queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
uint64_t & downloadSize_, uint64_t & narSize_)
{
@ -156,7 +156,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
std::function<void(StorePathWithOutputs)> doPath;
std::function<void(BuildableReq)> doPath;
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
{
@ -165,7 +165,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
}
for (auto & i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second }));
pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second }));
};
auto checkOutput = [&](
@ -188,24 +188,25 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
drvState->outPaths.insert(outPath);
if (!drvState->left) {
for (auto & path : drvState->outPaths)
pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } ));
pool.enqueue(std::bind(doPath, BuildableOpaque { path } ));
}
}
}
};
doPath = [&](const StorePathWithOutputs & path) {
doPath = [&](const BuildableReq & req) {
{
auto state(state_.lock());
if (!state->done.insert(path.to_string(*this)).second) return;
if (!state->done.insert(req.to_string(*this)).second) return;
}
if (path.path.isDerivation()) {
if (!isValidPath(path.path)) {
std::visit(overloaded {
[&](BuildableReqFromDrv bfd) {
if (!isValidPath(bfd.drvPath)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
state->unknown.insert(path.path);
state->unknown.insert(bfd.drvPath);
return;
}
@ -213,52 +214,54 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) {
if (!pathOpt) {
knownOutputPaths = false;
break;
}
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt))
invalid.insert(*pathOpt);
}
if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(path.path));
ParsedDerivation parsedDrv(StorePath(path.path), *drv);
auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath));
ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv);
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState));
pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState));
} else
mustBuildDrv(path.path, *drv);
mustBuildDrv(bfd.drvPath, *drv);
} else {
},
[&](BuildableOpaque bo) {
if (isValidPath(path.path)) return;
if (isValidPath(bo.path)) return;
SubstitutablePathInfos infos;
querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos);
if (infos.empty()) {
auto state(state_.lock());
state->unknown.insert(path.path);
state->unknown.insert(bo.path);
return;
}
auto info = infos.find(path.path);
auto info = infos.find(bo.path);
assert(info != infos.end());
{
auto state(state_.lock());
state->willSubstitute.insert(path.path);
state->willSubstitute.insert(bo.path);
state->downloadSize += info->second.downloadSize;
state->narSize += info->second.narSize;
}
for (auto & ref : info->second.references)
pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref }));
}
pool.enqueue(std::bind(doPath, BuildableOpaque { ref }));
},
}, req.raw());
};
for (auto & path : targets)

71
src/libstore/path-with-outputs.cc Normal file
View file

@ -0,0 +1,71 @@
#include "path-with-outputs.hh"
#include "store-api.hh"
namespace nix {
std::string StorePathWithOutputs::to_string(const Store & store) const
{
return outputs.empty()
? store.printStorePath(path)
: store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
}
BuildableReq StorePathWithOutputs::toBuildableReq() const
{
if (!outputs.empty() || path.isDerivation())
return BuildableReqFromDrv { path, outputs };
else
return BuildableOpaque { path };
}
std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs> ss)
{
std::vector<BuildableReq> reqs;
for (auto & s : ss) reqs.push_back(s.toBuildableReq());
return reqs;
}
std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p)
{
return std::visit(overloaded {
[&](BuildableOpaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
if (bo.path.isDerivation()) {
// drv path gets interpreted as "build", not "get drv file itself"
return bo.path;
}
return StorePathWithOutputs { bo.path };
},
[&](BuildableReqFromDrv bfd) -> std::variant<StorePathWithOutputs, StorePath> {
return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
},
}, p.raw());
}
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
}
StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
{
auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) };
}
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
{
auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
}
}

35
src/libstore/path-with-outputs.hh Normal file
View file

@ -0,0 +1,35 @@
#pragma once
#include <variant>
#include "path.hh"
#include "buildable.hh"
namespace nix {
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
std::string to_string(const Store & store) const;
BuildableReq toBuildableReq() const;
static std::variant<StorePathWithOutputs, StorePath> tryFromBuildableReq(const BuildableReq &);
};
std::vector<BuildableReq> toBuildableReqs(const std::vector<StorePathWithOutputs>);
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
class Store;
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
}

15
src/libstore/path.cc
View file

@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const
return res;
}
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
}
StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
{
auto [path, outputs] = nix::parsePathWithOutputs(s);
return {parseStorePath(path), std::move(outputs)};
}
}

10
src/libstore/path.hh
View file

@ -68,16 +68,6 @@ typedef std::map<string, StorePath> OutputPathMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
std::string to_string(const Store & store) const;
};
std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
}
namespace std {

57
src/libstore/remote-store.cc
View file

@ -1,5 +1,6 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "remote-fs-accessor.hh"
#include "remote-store.hh"
#include "worker-protocol.hh"
@ -50,6 +51,19 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
out << renderContentAddress(ca);
}
BuildableReq read(const Store & store, Source & from, Phantom<BuildableReq> _)
{
auto s = readString(from);
return BuildableReq::parse(store, s);
}
void write(const Store & store, Sink & out, const BuildableReq & req)
{
out << req.to_string(store);
}
Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
{
std::string rawInput = readString(from);
@ -58,8 +72,12 @@ Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
"remote-protocol"
);
}
void write(const Store & store, Sink & out, const Realisation & realisation)
{ out << realisation.toJSON().dump(); }
{
out << realisation.toJSON().dump();
}
DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
{
@ -657,16 +675,36 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
return {Realisation{.id = id, .outPath = *outPaths.begin()}};
}
static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector<BuildableReq> & reqs)
{
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
worker_proto::write(store, conn->to, reqs);
} else {
Strings ss;
for (auto & p : reqs) {
auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p);
std::visit(overloaded {
[&](StorePathWithOutputs s) {
ss.push_back(s.to_string(store));
},
[&](StorePath drvPath) {
throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
store.printStorePath(drvPath),
GET_PROTOCOL_MAJOR(conn->daemonVersion),
GET_PROTOCOL_MINOR(conn->daemonVersion));
},
}, sOrDrvPath);
}
conn->to << ss;
}
}
void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
void RemoteStore::buildPaths(const std::vector<BuildableReq> & drvPaths, BuildMode buildMode)
{
auto conn(getConnection());
conn->to << wopBuildPaths;
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
Strings ss;
for (auto & p : drvPaths)
ss.push_back(p.to_string(*this));
conn->to << ss;
writeBuildableReqs(*this, conn, drvPaths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
conn->to << buildMode;
else
@ -805,7 +843,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
}
void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
void RemoteStore::queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize)
{
@ -816,10 +854,7 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
// to prevent a deadlock.
goto fallback;
conn->to << wopQueryMissing;
Strings ss;
for (auto & p : targets)
ss.push_back(p.to_string(*this));
conn->to << ss;
writeBuildableReqs(*this, conn, targets);
conn.processStderr();
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});

4
src/libstore/remote-store.hh
View file

@ -86,7 +86,7 @@ public:
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
void buildPaths(const std::vector<BuildableReq> & paths, BuildMode buildMode) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
@ -109,7 +109,7 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
void queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override;

15
src/libstore/store-api.cc
View file

@ -54,13 +54,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
}
StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
{
auto [path2, outputs] = nix::parsePathWithOutputs(path);
return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
}
/* Store paths have the following form:
<realized-path> = <store>/<h>-<name>
@ -552,10 +545,10 @@ void Store::queryPathInfo(const StorePath & storePath,
void Store::substitutePaths(const StorePathSet & paths)
{
std::vector<StorePathWithOutputs> paths2;
std::vector<BuildableReq> paths2;
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back({path});
paths2.push_back(BuildableOpaque{path});
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
queryMissing(paths2,
@ -563,8 +556,8 @@ void Store::substitutePaths(const StorePathSet & paths)
if (!willSubstitute.empty())
try {
std::vector<StorePathWithOutputs> subs;
for (auto & p : willSubstitute) subs.push_back({p});
std::vector<BuildableReq> subs;
for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p});
buildPaths(subs);
} catch (Error & e) {
logWarning(e.info());

12
src/libstore/store-api.hh
View file

@ -2,6 +2,7 @@
#include "realisation.hh"
#include "path.hh"
#include "buildable.hh"
#include "hash.hh"
#include "content-address.hh"
#include "serialise.hh"
@ -263,11 +264,6 @@ public:
PathSet printStorePathSet(const StorePathSet & path) const;
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
StorePathWithOutputs parsePathWithOutputs(const string & s);
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
std::string showPaths(const StorePathSet & paths);
@ -291,8 +287,6 @@ public:
result. */
StorePath followLinksToStorePath(std::string_view path) const;
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
/* Constructs a unique store path name. */
StorePath makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const;
@ -496,7 +490,7 @@ public:
recursively building any sub-derivations. For inputs that are
not derivations, substitute them. */
virtual void buildPaths(
const std::vector<StorePathWithOutputs> & paths,
const std::vector<BuildableReq> & paths,
BuildMode buildMode = bmNormal);
/* Build a single non-materialized derivation (i.e. not from an
@ -658,7 +652,7 @@ public:
/* Given a set of paths that are to be built, return the set of
derivations that will be built, and the set of output paths
that will be substituted. */
virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
virtual void queryMissing(const std::vector<BuildableReq> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize);

22
src/libstore/worker-protocol.hh
View file

@ -86,9 +86,11 @@ namespace worker_proto {
MAKE_WORKER_PROTO(, std::string);
MAKE_WORKER_PROTO(, StorePath);
MAKE_WORKER_PROTO(, ContentAddress);
MAKE_WORKER_PROTO(, BuildableReq);
MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
#define X_ template<typename K, typename V>
@ -113,6 +115,26 @@ MAKE_WORKER_PROTO(X_, Y_);
MAKE_WORKER_PROTO(, std::optional<StorePath>);
MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
template<typename T>
std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
{
std::vector<T> resSet;
auto size = readNum<size_t>(from);
while (size--) {
resSet.push_back(read(store, from, Phantom<T> {}));
}
return resSet;
}
template<typename T>
void write(const Store & store, Sink & out, const std::vector<T> & resSet)
{
out << resSet.size();
for (auto & key : resSet) {
write(store, out, key);
}
}
template<typename T>
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
{