nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-21 17:59:36 +01:00
Code Activity

Merge branch 'read-only-local-store' into overlayfs-store

Browse source
This commit is contained in:
Ben Radford 2023-05-23 09:52:41 +01:00
commit ff12cf3b94
No known key found for this signature in database
GPG key ID: 9DF5D4640AB888D5
91 changed files with 975 additions and 545 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/libstore/build/derivation-goal.cc
View file

@ -1152,7 +1152,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Tell the hook all the inputs that have to be copied to the
remote system. */
worker_proto::write(worker.store, hook->sink, inputPaths);
workerProtoWrite(worker.store, hook->sink, inputPaths);
/* Tell the hooks the missing outputs that have to be copied back
from the remote system. */
@ -1163,7 +1163,7 @@ HookReply DerivationGoal::tryBuildHook()
if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
missingOutputs.insert(outputName);
}
worker_proto::write(worker.store, hook->sink, missingOutputs);
workerProtoWrite(worker.store, hook->sink, missingOutputs);
}
hook->sink = FdSink();

2
src/libstore/build/entry-points.cc
View file

@ -110,7 +110,7 @@ void Store::ensurePath(const StorePath & path)
}
void LocalStore::repairPath(const StorePath & path)
void Store::repairPath(const StorePath & path)
{
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);

2
src/libstore/build/local-derivation-goal.cc
View file

@ -1771,6 +1771,8 @@ void LocalDerivationGoal::runChild()
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
if (pathExists(path))
ss.push_back(path);
dirsInChroot.emplace(settings.caFile, "/etc/ssl/certs/ca-certificates.crt");
}
for (auto & i : ss) dirsInChroot.emplace(i, i);

50
src/libstore/daemon.cc
View file

@ -263,7 +263,7 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli
{
std::vector<DerivedPath> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
reqs = WorkerProto<std::vector<DerivedPath>>::read(store, from);
} else {
for (auto & s : readStrings<Strings>(from))
reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
@ -287,7 +287,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryValidPaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, from);
SubstituteFlag substitute = NoSubstitute;
if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
@ -300,7 +300,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
auto res = store->queryValidPaths(paths, substitute);
logger->stopWork();
worker_proto::write(*store, to, res);
workerProtoWrite(*store, to, res);
break;
}
@ -316,11 +316,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, from);
logger->startWork();
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
worker_proto::write(*store, to, res);
workerProtoWrite(*store, to, res);
break;
}
@ -349,7 +349,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
paths = store->queryValidDerivers(path);
else paths = store->queryDerivationOutputs(path);
logger->stopWork();
worker_proto::write(*store, to, paths);
workerProtoWrite(*store, to, paths);
break;
}
@ -367,7 +367,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto outputs = store->queryPartialDerivationOutputMap(path);
logger->stopWork();
worker_proto::write(*store, to, outputs);
workerProtoWrite(*store, to, outputs);
break;
}
@ -393,7 +393,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
auto name = readString(from);
auto camStr = readString(from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto refs = WorkerProto<StorePathSet>::read(*store, from);
bool repairBool;
from >> repairBool;
auto repair = RepairFlag{repairBool};
@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopAddTextToStore: {
std::string suffix = readString(from);
std::string s = readString(from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto refs = WorkerProto<StorePathSet>::read(*store, from);
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
logger->stopWork();
@ -567,7 +567,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto results = store->buildPathsWithResults(drvs, mode);
logger->stopWork();
worker_proto::write(*store, to, results);
workerProtoWrite(*store, to, results);
break;
}
@ -644,7 +644,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(*store, to, builtOutputs);
workerProtoWrite(*store, to, builtOutputs);
}
break;
}
@ -709,7 +709,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopCollectGarbage: {
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
from >> options.ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
@ -779,7 +779,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
else {
to << 1
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
worker_proto::write(*store, to, i->second.references);
workerProtoWrite(*store, to, i->second.references);
to << i->second.downloadSize
<< i->second.narSize;
}
@ -790,11 +790,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
SubstitutablePathInfos infos;
StorePathCAMap pathsMap = {};
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, from);
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
} else
pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
logger->startWork();
store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
@ -802,7 +802,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
for (auto & i : infos) {
to << store->printStorePath(i.first)
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
worker_proto::write(*store, to, i.second.references);
workerProtoWrite(*store, to, i.second.references);
to << i.second.downloadSize << i.second.narSize;
}
break;
@ -812,7 +812,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto paths = store->queryAllValidPaths();
logger->stopWork();
worker_proto::write(*store, to, paths);
workerProtoWrite(*store, to, paths);
break;
}
@ -884,7 +884,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
info.ca = ContentAddress::parseOpt(readString(from));
@ -935,9 +935,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
worker_proto::write(*store, to, willBuild);
worker_proto::write(*store, to, willSubstitute);
worker_proto::write(*store, to, unknown);
workerProtoWrite(*store, to, willBuild);
workerProtoWrite(*store, to, willSubstitute);
workerProtoWrite(*store, to, unknown);
to << downloadSize << narSize;
break;
}
@ -950,7 +950,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
} else {
auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
auto realisation = WorkerProto<Realisation>::read(*store, from);
store->registerDrvOutput(realisation);
}
logger->stopWork();
@ -965,11 +965,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
workerProtoWrite(*store, to, outPaths);
} else {
std::set<Realisation> realisations;
if (info) realisations.insert(*info);
worker_proto::write(*store, to, realisations);
workerProtoWrite(*store, to, realisations);
}
break;
}
@ -1045,7 +1045,7 @@ void processConnection(
auto temp = trusted
? store->isTrustedClient()
: std::optional { NotTrusted };
worker_proto::write(*store, to, temp);
workerProtoWrite(*store, to, temp);
}
/* Send startup error messages to the client. */

15
src/libstore/derivations.cc
View file

@ -1,4 +1,5 @@
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "store-api.hh"
#include "globals.hh"
#include "util.hh"
@ -748,7 +749,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);
@ -796,7 +797,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
},
}, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
workerProtoWrite(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)
@ -810,13 +811,7 @@ std::string hashPlaceholder(const std::string_view outputName)
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
}
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
}
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
@ -880,7 +875,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
for (auto & outputName : inputOutputs) {
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
store.printStorePath(*actualPath));
resolved.inputSrcs.insert(*actualPath);
} else {

12
src/libstore/derivations.hh
View file

@ -6,6 +6,7 @@
#include "hash.hh"
#include "content-address.hh"
#include "repair-flag.hh"
#include "derived-path.hh"
#include "sync.hh"
#include "comparator.hh"
@ -495,17 +496,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
*/
std::string hashPlaceholder(const std::string_view outputName);
/**
* This creates an opaque and almost certainly unique string
* deterministically from a derivation path and output name.
*
* It is used as a placeholder to allow derivations to refer to
* content-addressed paths whose content --- and thus the path
* themselves --- isn't yet known. This occurs when a derivation has a
* dependency which is a CA derivation.
*/
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
extern const Hash impureOutputHash;
}

39
src/libstore/downstream-placeholder.cc Normal file
View file

@ -0,0 +1,39 @@
#include "downstream-placeholder.hh"
#include "derivations.hh"
namespace nix {
std::string DownstreamPlaceholder::render() const
{
return "/" + hash.to_string(Base32, false);
}
DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
const StorePath & drvPath,
std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return DownstreamPlaceholder {
hashString(htSHA256, clearText)
};
}
DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
const DownstreamPlaceholder & placeholder,
std::string_view outputName,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::DynamicDerivations);
auto compressed = compressHash(placeholder.hash, 20);
auto clearText = "nix-computed-output:"
+ compressed.to_string(Base32, false)
+ ":" + std::string { outputName };
return DownstreamPlaceholder {
hashString(htSHA256, clearText)
};
}
}

75
src/libstore/downstream-placeholder.hh Normal file
View file

@ -0,0 +1,75 @@
#pragma once
///@file
#include "hash.hh"
#include "path.hh"
namespace nix {
/**
* Downstream Placeholders are opaque and almost certainly unique values
* used to allow derivations to refer to store objects which are yet to
* be built and for we do not yet have store paths for.
*
* They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`,
* except for the cases involving input addressing or fixed outputs
* where we do know a store path for the derivation output in advance.
*
* Unlike `DerivationPath`, however, `DownstreamPlaceholder` is
* purposefully opaque and obfuscated. This is so they are hard to
* create by accident, and so substituting them (once we know what the
* path to store object is) is unlikely to capture other stuff it
* shouldn't.
*
* We use them with `Derivation`: the `render()` method is called to
* render an opaque string which can be used in the derivation, and the
* resolving logic can substitute those strings for store paths when
* resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`.
*/
class DownstreamPlaceholder
{
/**
* `DownstreamPlaceholder` is just a newtype of `Hash`.
* This its only field.
*/
Hash hash;
/**
* Newtype constructor
*/
DownstreamPlaceholder(Hash hash) : hash(hash) { }
public:
/**
* This creates an opaque and almost certainly unique string
* deterministically from the placeholder.
*/
std::string render() const;
/**
* Create a placeholder for an unknown output of a content-addressed
* derivation.
*
* The derivation itself is known (we have a store path for it), but
* the output doesn't yet have a known store path.
*/
static DownstreamPlaceholder unknownCaOutput(
const StorePath & drvPath,
std::string_view outputName);
/**
* Create a placehold for the output of an unknown derivation.
*
* The derivation is not yet known because it is a dynamic
* derivaiton --- it is itself an output of another derivation ---
* and we just have (another) placeholder for it.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DownstreamPlaceholder unknownDerivation(
const DownstreamPlaceholder & drvPlaceholder,
std::string_view outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
};
}

4
src/libstore/export-import.cc
View file

@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
teeSink
<< exportMagic
<< printStorePath(path);
worker_proto::write(*this, teeSink, info->references);
workerProtoWrite(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
//Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
auto references = WorkerProto<StorePathSet>::read(*this, source);
auto deriver = readString(source);
auto narHash = hashString(htSHA256, saved.s);

5
src/libstore/gc.cc
View file

@ -110,6 +110,11 @@ void LocalStore::createTempRootsFile()
void LocalStore::addTempRoot(const StorePath & path)
{
if (readOnly) {
debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways.");
return;
}
createTempRootsFile();
/* Open/create the global GC lock file. */

27
src/libstore/legacy-ssh-store.cc
View file

@ -146,7 +146,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
auto deriver = readString(conn->from);
if (deriver != "")
info->deriver = parseStorePath(deriver);
info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
readLongLong(conn->from); // download size
info->narSize = readLongLong(conn->from);
@ -180,7 +180,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
worker_proto::write(*this, conn->to, info.references);
workerProtoWrite(*this, conn->to, info.references);
conn->to
<< info.registrationTime
<< info.narSize
@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
conn->to
<< exportMagic
<< printStorePath(info.path);
worker_proto::write(*this, conn->to, info.references);
workerProtoWrite(*this, conn->to, info.references);
conn->to
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0
@ -294,7 +294,7 @@ public:
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
for (auto && [output, realisation] : builtOutputs)
status.builtOutputs.insert_or_assign(
std::move(output.outputName),
@ -344,6 +344,17 @@ public:
virtual ref<FSAccessor> getFSAccessor() override
{ unsupported("getFSAccessor"); }
/**
* The default instance would schedule the work on the client side, but
* for consistency with `buildPaths` and `buildDerivation` it should happen
* on the remote side.
*
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override
@ -358,10 +369,10 @@ public:
conn->to
<< cmdQueryClosure
<< includeOutputs;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
conn->to.flush();
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
out.insert(i);
}
@ -374,10 +385,10 @@ public:
<< cmdQueryValidPaths
<< false // lock
<< maybeSubstitute;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
conn->to.flush();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
void connect() override

43
src/libstore/local-store.cc
View file

@ -202,10 +202,16 @@ LocalStore::LocalStore(const Params & params)
createSymlink(profilesDir, gcRootsDir + "/profiles");
}
if (readOnly) {
experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore);
}
for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
createDirs(perUserDir);
if (chmod(perUserDir.c_str(), 0755) == -1)
throw SysError("could not set permissions on '%s' to 755", perUserDir);
if (!readOnly) {
if (chmod(perUserDir.c_str(), 0755) == -1)
throw SysError("could not set permissions on '%s' to 755", perUserDir);
}
}
/* Optionally, create directories and set permissions for a
@ -269,10 +275,12 @@ LocalStore::LocalStore(const Params & params)
/* Acquire the big fat lock in shared mode to make sure that no
schema upgrade is in progress. */
Path globalLockPath = dbDir + "/big-lock";
globalLock = openLockFile(globalLockPath.c_str(), true);
if (!readOnly) {
Path globalLockPath = dbDir + "/big-lock";
globalLock = openLockFile(globalLockPath.c_str(), true);
}
if (!lockFile(globalLock.get(), ltRead, false)) {
if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) {
printInfo("waiting for the big Nix store lock...");
lockFile(globalLock.get(), ltRead, true);
}
@ -280,6 +288,14 @@ LocalStore::LocalStore(const Params & params)
/* Check the current database schema and if necessary do an
upgrade. */
int curSchema = getSchema();
if (readOnly && curSchema < nixSchemaVersion) {
debug("current schema version: %d", curSchema);
debug("supported schema version: %d", nixSchemaVersion);
throw Error(curSchema == 0 ?
"database does not exist, and cannot be created in read-only mode" :
"database schema needs migrating, but this cannot be done in read-only mode");
}
if (curSchema > nixSchemaVersion)
throw Error("current Nix store schema is version %1%, but I only support %2%",
curSchema, nixSchemaVersion);
@ -344,7 +360,11 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false);
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
if (!readOnly) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
} else {
throw Error("need to migrate to CA schema, but this cannot be done in read-only mode");
}
}
/* Prepare SQL statements. */
@ -475,13 +495,20 @@ int LocalStore::getSchema()
void LocalStore::openDB(State & state, bool create)
{
if (access(dbDir.c_str(), R_OK | W_OK))
if (create && readOnly) {
throw Error("unable to create database while in read-only mode");
}
if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK)))
throw SysError("Nix database directory '%1%' is not writable", dbDir);
/* Open the Nix database. */
std::string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
state.db = SQLite(dbPath, create);
auto openMode = readOnly ? SQLiteOpenMode::Immutable
: create ? SQLiteOpenMode::Normal
: SQLiteOpenMode::NoCreate;
state.db = SQLite(dbPath, openMode);
#ifdef __CYGWIN__
/* The cygwin version of sqlite3 has a patch which calls

18
src/libstore/local-store.hh
View file

@ -46,6 +46,22 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
"require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
Setting<bool> readOnly{(StoreConfig*) this,
false,
"read-only",
R"(
Allow this store to be opened when its database is on a read-only filesystem.
Normally Nix will attempt to open the store database in read-write mode, even
for querying (when write access is not needed). This causes it to fail if the
database is on a read-only filesystem.
Enable read-only mode to disable locking and open the SQLite database with the
**immutable** parameter set. Do not use this unless the filesystem is read-only.
Using it when the filesystem is writable can cause incorrect query results or
corruption errors if the database is changed by another process.
)"};
const std::string name() override { return "Local Store"; }
std::string doc() override;
@ -240,8 +256,6 @@ public:
void vacuumDB();
void repairPath(const StorePath & path) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
/**

5
src/libstore/path-info.cc
View file

@ -1,5 +1,6 @@
#include "path-info.hh"
#include "worker-protocol.hh"
#include "store-api.hh"
namespace nix {
@ -131,7 +132,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
auto narHash = Hash::parseAny(readString(source), htSHA256);
ValidPathInfo info(path, narHash);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
info.references = worker_proto::read(store, source, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(store, source);
source >> info.registrationTime >> info.narSize;
if (format >= 16) {
source >> info.ultimate;
@ -152,7 +153,7 @@ void ValidPathInfo::write(
sink << store.printStorePath(path);
sink << (deriver ? store.printStorePath(*deriver) : "")
<< narHash.to_string(Base16, false);
worker_proto::write(store, sink, references);
workerProtoWrite(store, sink, references);
sink << registrationTime << narSize;
if (format >= 16) {
sink << ultimate

245
src/libstore/remote-store.cc
View file

@ -18,189 +18,6 @@
namespace nix {
namespace worker_proto {
std::string read(const Store & store, Source & from, Phantom<std::string> _)
{
return readString(from);
}
void write(const Store & store, Sink & out, const std::string & str)
{
out << str;
}
StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
{
return store.parseStorePath(readString(from));
}
void write(const Store & store, Sink & out, const StorePath & storePath)
{
out << store.printStorePath(storePath);
}
std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
{
auto temp = readNum<uint8_t>(from);
switch (temp) {
case 0:
return std::nullopt;
case 1:
return { Trusted };
case 2:
return { NotTrusted };
default:
throw Error("Invalid trusted status from remote");
}
}
void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
{
if (!optTrusted)
out << (uint8_t)0;
else {
switch (*optTrusted) {
case Trusted:
out << (uint8_t)1;
break;
case NotTrusted:
out << (uint8_t)2;
break;
default:
assert(false);
};
}
}
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
{
return ContentAddress::parse(readString(from));
}
void write(const Store & store, Sink & out, const ContentAddress & ca)
{
out << renderContentAddress(ca);
}
DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
{
auto s = readString(from);
return DerivedPath::parseLegacy(store, s);
}
void write(const Store & store, Sink & out, const DerivedPath & req)
{
out << req.to_string_legacy(store);
}
Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
{
std::string rawInput = readString(from);
return Realisation::fromJSON(
nlohmann::json::parse(rawInput),
"remote-protocol"
);
}
void write(const Store & store, Sink & out, const Realisation & realisation)
{
out << realisation.toJSON().dump();
}
DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
{
return DrvOutput::parse(readString(from));
}
void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
{
out << drvOutput.to_string();
}
KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
{
auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
auto br = worker_proto::read(store, from, Phantom<BuildResult> {});
return KeyedBuildResult {
std::move(br),
/* .path = */ std::move(path),
};
}
void write(const Store & store, Sink & to, const KeyedBuildResult & res)
{
worker_proto::write(store, to, res.path);
worker_proto::write(store, to, static_cast<const BuildResult &>(res));
}
BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
{
BuildResult res;
res.status = (BuildResult::Status) readInt(from);
from
>> res.errorMsg
>> res.timesBuilt
>> res.isNonDeterministic
>> res.startTime
>> res.stopTime;
auto builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
return res;
}
void write(const Store & store, Sink & to, const BuildResult & res)
{
to
<< res.status
<< res.errorMsg
<< res.timesBuilt
<< res.isNonDeterministic
<< res.startTime
<< res.stopTime;
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(store, to, builtOutputs);
}
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{
auto s = readString(from);
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
}
void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
{
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
}
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
{
return ContentAddress::parseOpt(readString(from));
}
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
{
out << (caOpt ? renderContentAddress(*caOpt) : "");
}
}
/* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
: RemoteStoreConfig(params)
@ -283,7 +100,7 @@ void RemoteStore::initConnection(Connection & conn)
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {});
conn.remoteTrustsUs = WorkerProto<std::optional<TrustedFlag>>::read(*this, conn.from);
} else {
// We don't know the answer; protocol to old.
conn.remoteTrustsUs = std::nullopt;
@ -410,12 +227,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
return res;
} else {
conn->to << wopQueryValidPaths;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
}
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
}
@ -425,7 +242,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
auto conn(getConnection());
conn->to << wopQueryAllValidPaths;
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
@ -442,9 +259,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
return res;
} else {
conn->to << wopQuerySubstitutablePaths;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
}
@ -466,7 +283,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
infos.insert_or_assign(i.first, std::move(info));
@ -479,9 +296,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
StorePathSet paths;
for (auto & path : pathsMap)
paths.insert(path.first);
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
} else
worker_proto::write(*this, conn->to, pathsMap);
workerProtoWrite(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
@ -489,7 +306,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
}
@ -532,7 +349,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
auto conn(getConnection());
conn->to << wopQueryReferrers << printStorePath(path);
conn.processStderr();
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
referrers.insert(i);
}
@ -542,7 +359,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
auto conn(getConnection());
conn->to << wopQueryValidDerivers << printStorePath(path);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
@ -554,7 +371,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
auto conn(getConnection());
conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
@ -564,7 +381,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
auto conn(getConnection());
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
} else {
// Fallback for old daemon versions.
// For floating-CA derivations (and their co-dependencies) this is an
@ -610,7 +427,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
<< wopAddToStore
<< name
<< caMethod.render(hashType);
worker_proto::write(*this, conn->to, references);
workerProtoWrite(*this, conn->to, references);
conn->to << repair;
// The dump source may invoke the store, so we need to make some room.
@ -635,7 +452,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
name, printHashType(hashType));
std::string s = dump.drain();
conn->to << wopAddTextToStore << name << s;
worker_proto::write(*this, conn->to, references);
workerProtoWrite(*this, conn->to, references);
conn.processStderr();
},
[&](const FileIngestionMethod & fim) -> void {
@ -701,7 +518,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
sink
<< exportMagic
<< printStorePath(info.path);
worker_proto::write(*this, sink, info.references);
workerProtoWrite(*this, sink, info.references);
sink
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0 // == no legacy signature
@ -711,7 +528,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn.processStderr(0, source2.get());
auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
assert(importedPaths.size() <= 1);
}
@ -720,7 +537,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
worker_proto::write(*this, conn->to, info.references);
workerProtoWrite(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
@ -793,7 +610,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
conn->to << info.id.to_string();
conn->to << std::string(info.outPath.to_string());
} else {
worker_proto::write(*this, conn->to, info);
workerProtoWrite(*this, conn->to, info);
}
conn.processStderr();
}
@ -815,14 +632,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
auto real = [&]() -> std::shared_ptr<const Realisation> {
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
auto outPaths = worker_proto::read(
*this, conn->from, Phantom<std::set<StorePath>> {});
auto outPaths = WorkerProto<std::set<StorePath>>::read(
*this, conn->from);
if (outPaths.empty())
return nullptr;
return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
} else {
auto realisations = worker_proto::read(
*this, conn->from, Phantom<std::set<Realisation>> {});
auto realisations = WorkerProto<std::set<Realisation>>::read(
*this, conn->from);
if (realisations.empty())
return nullptr;
return std::make_shared<const Realisation>(*realisations.begin());
@ -836,7 +653,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
{
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
worker_proto::write(store, conn->to, reqs);
workerProtoWrite(store, conn->to, reqs);
} else {
Strings ss;
for (auto & p : reqs) {
@ -906,7 +723,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
writeDerivedPaths(*this, conn, paths);
conn->to << buildMode;
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<std::vector<KeyedBuildResult>> {});
return WorkerProto<std::vector<KeyedBuildResult>>::read(*this, conn->from);
} else {
// Avoid deadlock.
conn_.reset();
@ -989,7 +806,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
@ -1048,7 +865,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
conn->to
<< wopCollectGarbage << options.action;
worker_proto::write(*this, conn->to, options.pathsToDelete);
workerProtoWrite(*this, conn->to, options.pathsToDelete);
conn->to << options.ignoreLiveness
<< options.maxFreed
/* removed options */
@ -1107,9 +924,9 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
conn->to << wopQueryMissing;
writeDerivedPaths(*this, conn, targets);
conn.processStderr();
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
conn->from >> downloadSize >> narSize;
return;
}

11
src/libstore/remote-store.hh
View file

@ -137,6 +137,17 @@ public:
bool verifyStore(bool checkContents, RepairFlag repair) override;
/**
* The default instance would schedule the work on the client side, but
* for consistency with `buildPaths` and `buildDerivation` it should happen
* on the remote side.
*
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
void queryMissing(const std::vector<DerivedPath> & targets,

11
src/libstore/sqlite.cc
View file

@ -1,6 +1,7 @@
#include "sqlite.hh"
#include "globals.hh"
#include "util.hh"
#include "url.hh"
#include <sqlite3.h>
@ -50,15 +51,17 @@ static void traceSQL(void * x, const char * sql)
notice("SQL<[%1%]>", sql);
};
SQLite::SQLite(const Path & path, bool create)
SQLite::SQLite(const Path & path, SQLiteOpenMode mode)
{
// useSQLiteWAL also indicates what virtual file system we need. Using
// `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
// for Linux (WSL) where useSQLiteWAL should be false by default.
const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
int flags = SQLITE_OPEN_READWRITE;
if (create) flags |= SQLITE_OPEN_CREATE;
int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs);
bool immutable = mode == SQLiteOpenMode::Immutable;
int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE;
if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE;
auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0");
int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs);
if (ret != SQLITE_OK) {
const char * err = sqlite3_errstr(ret);
throw Error("cannot open SQLite database '%s': %s", path, err);

23
src/libstore/sqlite.hh
View file

@ -11,6 +11,27 @@ struct sqlite3_stmt;
namespace nix {
enum class SQLiteOpenMode {
/**
* Open the database in read-write mode.
* If the database does not exist, it will be created.
*/
Normal,
/**
* Open the database in read-write mode.
* Fails with an error if the database does not exist.
*/
NoCreate,
/**
* Open the database in immutable mode.
* In addition to the database being read-only,
* no wal or journal files will be created by sqlite.
* Use this mode if the database is on a read-only filesystem.
* Fails with an error if the database does not exist.
*/
Immutable
};
/**
* RAII wrapper to close a SQLite database automatically.
*/
@ -18,7 +39,7 @@ struct SQLite
{
sqlite3 * db = 0;
SQLite() { }
SQLite(const Path & path, bool create = true);
SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal);
SQLite(const SQLite & from) = delete;
SQLite& operator = (const SQLite & from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }

23
src/libstore/ssh.cc
View file

@ -41,6 +41,11 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
args.push_back("-oLocalCommand=echo started");
}
bool SSHMaster::isMasterRunning() {
auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true});
return res.first == 0;
}
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
{
Path socketPath = startMaster();
@ -97,7 +102,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
// Wait for the SSH connection to be established,
// So that we don't overwrite the password prompt with our progress bar.
if (!fakeSSH && !useMaster) {
if (!fakeSSH && !useMaster && !isMasterRunning()) {
std::string reply;
try {
reply = readLine(out.readSide.get());
@ -133,6 +138,8 @@ Path SSHMaster::startMaster()
logger->pause();
Finally cleanup = [&]() { logger->resume(); };
bool wasMasterRunning = isMasterRunning();
state->sshMaster = startProcess([&]() {
restoreProcessContext();
@ -152,13 +159,15 @@ Path SSHMaster::startMaster()
out.writeSide = -1;
std::string reply;
try {
reply = readLine(out.readSide.get());
} catch (EndOfFile & e) { }
if (!wasMasterRunning) {
std::string reply;
try {
reply = readLine(out.readSide.get());
} catch (EndOfFile & e) { }
if (reply != "started")
throw Error("failed to start SSH master connection to '%s'", host);
if (reply != "started")
throw Error("failed to start SSH master connection to '%s'", host);
}
return state->socketPath;
}

1
src/libstore/ssh.hh
View file

@ -28,6 +28,7 @@ private:
Sync<State> state_;
void addCommonSSHOpts(Strings & args);
bool isMasterRunning();
public:

3
src/libstore/store-api.hh
View file

@ -679,8 +679,7 @@ public:
* Repair the contents of the given path by redownloading it using
* a substituter (if available).
*/
virtual void repairPath(const StorePath & path)
{ unsupported("repairPath"); }
virtual void repairPath(const StorePath & path);
/**
* Add signatures to the specified store path. The signatures are

33
src/libstore/tests/downstream-placeholder.cc Normal file
View file

@ -0,0 +1,33 @@
#include <gtest/gtest.h>
#include "downstream-placeholder.hh"
namespace nix {
TEST(DownstreamPlaceholder, unknownCaOutput) {
ASSERT_EQ(
DownstreamPlaceholder::unknownCaOutput(
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
"out").render(),
"/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz");
}
TEST(DownstreamPlaceholder, unknownDerivation) {
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
ASSERT_EQ(
DownstreamPlaceholder::unknownDerivation(
DownstreamPlaceholder::unknownCaOutput(
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" },
"out"),
"out",
mockXpSettings).render(),
"/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w");
}
}

192
src/libstore/worker-protocol.cc Normal file
View file

@ -0,0 +1,192 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "build-result.hh"
#include "worker-protocol.hh"
#include "archive.hh"
#include "derivations.hh"
#include <nlohmann/json.hpp>
namespace nix {
std::string WorkerProto<std::string>::read(const Store & store, Source & from)
{
return readString(from);
}
void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
{
out << str;
}
StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
{
return store.parseStorePath(readString(from));
}
void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
{
out << store.printStorePath(storePath);
}
std::optional<TrustedFlag> WorkerProto<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
{
auto temp = readNum<uint8_t>(from);
switch (temp) {
case 0:
return std::nullopt;
case 1:
return { Trusted };
case 2:
return { NotTrusted };
default:
throw Error("Invalid trusted status from remote");
}
}
void WorkerProto<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
{
if (!optTrusted)
out << (uint8_t)0;
else {
switch (*optTrusted) {
case Trusted:
out << (uint8_t)1;
break;
case NotTrusted:
out << (uint8_t)2;
break;
default:
assert(false);
};
}
}
ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
{
return ContentAddress::parse(readString(from));
}
void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
{
out << renderContentAddress(ca);
}
DerivedPath WorkerProto<DerivedPath>::read(const Store & store, Source & from)
{
auto s = readString(from);
return DerivedPath::parseLegacy(store, s);
}
void WorkerProto<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
{
out << req.to_string_legacy(store);
}
Realisation WorkerProto<Realisation>::read(const Store & store, Source & from)
{
std::string rawInput = readString(from);
return Realisation::fromJSON(
nlohmann::json::parse(rawInput),
"remote-protocol"
);
}
void WorkerProto<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
{
out << realisation.toJSON().dump();
}
DrvOutput WorkerProto<DrvOutput>::read(const Store & store, Source & from)
{
return DrvOutput::parse(readString(from));
}
void WorkerProto<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
{
out << drvOutput.to_string();
}
KeyedBuildResult WorkerProto<KeyedBuildResult>::read(const Store & store, Source & from)
{
auto path = WorkerProto<DerivedPath>::read(store, from);
auto br = WorkerProto<BuildResult>::read(store, from);
return KeyedBuildResult {
std::move(br),
/* .path = */ std::move(path),
};
}
void WorkerProto<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
{
workerProtoWrite(store, to, res.path);
workerProtoWrite(store, to, static_cast<const BuildResult &>(res));
}
BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
{
BuildResult res;
res.status = (BuildResult::Status) readInt(from);
from
>> res.errorMsg
>> res.timesBuilt
>> res.isNonDeterministic
>> res.startTime
>> res.stopTime;
auto builtOutputs = WorkerProto<DrvOutputs>::read(store, from);
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
return res;
}
void WorkerProto<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
{
to
<< res.status
<< res.errorMsg
<< res.timesBuilt
<< res.isNonDeterministic
<< res.startTime
<< res.stopTime;
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
workerProtoWrite(store, to, builtOutputs);
}
std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
{
auto s = readString(from);
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
}
void WorkerProto<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
{
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
}
std::optional<ContentAddress> WorkerProto<std::optional<ContentAddress>>::read(const Store & store, Source & from)
{
return ContentAddress::parseOpt(readString(from));
}
void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
{
out << (caOpt ? renderContentAddress(*caOpt) : "");
}
}

123
src/libstore/worker-protocol.hh
View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "store-api.hh"
#include "serialise.hh"
namespace nix {
@ -79,41 +78,81 @@ typedef enum {
class Store;
struct Source;
// items being serialized
struct DerivedPath;
struct DrvOutput;
struct Realisation;
struct BuildResult;
struct KeyedBuildResult;
enum TrustedFlag : bool;
/**
* Used to guide overloading
* Data type for canonical pairs of serializers for the worker protocol.
*
* See https://en.cppreference.com/w/cpp/language/adl for the broader
* concept of what is going on here.
*/
template<typename T>
struct Phantom {};
struct WorkerProto {
static T read(const Store & store, Source & from);
static void write(const Store & store, Sink & out, const T & t);
};
/**
* Wrapper function around `WorkerProto<T>::write` that allows us to
* infer the type instead of having to write it down explicitly.
*/
template<typename T>
void workerProtoWrite(const Store & store, Sink & out, const T & t)
{
WorkerProto<T>::write(store, out, t);
}
namespace worker_proto {
/* FIXME maybe move more stuff inside here */
/**
* Declare a canonical serializer pair for the worker protocol.
*
* We specialize the struct merely to indicate that we are implementing
* the function for the given type.
*
* Some sort of `template<...>` must be used with the caller for this to
* be legal specialization syntax. See below for what that looks like in
* practice.
*/
#define MAKE_WORKER_PROTO(T) \
struct WorkerProto< T > { \
static T read(const Store & store, Source & from); \
static void write(const Store & store, Sink & out, const T & t); \
};
#define MAKE_WORKER_PROTO(TEMPLATE, T) \
TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
TEMPLATE void write(const Store & store, Sink & out, const T & str)
template<>
MAKE_WORKER_PROTO(std::string);
template<>
MAKE_WORKER_PROTO(StorePath);
template<>
MAKE_WORKER_PROTO(ContentAddress);
template<>
MAKE_WORKER_PROTO(DerivedPath);
template<>
MAKE_WORKER_PROTO(Realisation);
template<>
MAKE_WORKER_PROTO(DrvOutput);
template<>
MAKE_WORKER_PROTO(BuildResult);
template<>
MAKE_WORKER_PROTO(KeyedBuildResult);
template<>
MAKE_WORKER_PROTO(std::optional<TrustedFlag>);
MAKE_WORKER_PROTO(, std::string);
MAKE_WORKER_PROTO(, StorePath);
MAKE_WORKER_PROTO(, ContentAddress);
MAKE_WORKER_PROTO(, DerivedPath);
MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(, BuildResult);
MAKE_WORKER_PROTO(, KeyedBuildResult);
MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
template<typename T>
MAKE_WORKER_PROTO(std::vector<T>);
template<typename T>
MAKE_WORKER_PROTO(std::set<T>);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
#define X_ template<typename K, typename V>
#define Y_ std::map<K, V>
MAKE_WORKER_PROTO(X_, Y_);
template<typename K, typename V>
#define X_ std::map<K, V>
MAKE_WORKER_PROTO(X_);
#undef X_
#undef Y_
/**
* These use the empty string for the null case, relying on the fact
@ -129,72 +168,72 @@ MAKE_WORKER_PROTO(X_, Y_);
* worker protocol harder to implement in other languages where such
* specializations may not be allowed.
*/
MAKE_WORKER_PROTO(, std::optional<StorePath>);
MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
template<>
MAKE_WORKER_PROTO(std::optional<StorePath>);
template<>
MAKE_WORKER_PROTO(std::optional<ContentAddress>);
template<typename T>
std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
{
std::vector<T> resSet;
auto size = readNum<size_t>(from);
while (size--) {
resSet.push_back(read(store, from, Phantom<T> {}));
resSet.push_back(WorkerProto<T>::read(store, from));
}
return resSet;
}
template<typename T>
void write(const Store & store, Sink & out, const std::vector<T> & resSet)
void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
{
out << resSet.size();
for (auto & key : resSet) {
write(store, out, key);
WorkerProto<T>::write(store, out, key);
}
}
template<typename T>
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
{
std::set<T> resSet;
auto size = readNum<size_t>(from);
while (size--) {
resSet.insert(read(store, from, Phantom<T> {}));
resSet.insert(WorkerProto<T>::read(store, from));
}
return resSet;
}
template<typename T>
void write(const Store & store, Sink & out, const std::set<T> & resSet)
void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
{
out << resSet.size();
for (auto & key : resSet) {
write(store, out, key);
WorkerProto<T>::write(store, out, key);
}
}
template<typename K, typename V>
std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
{
std::map<K, V> resMap;
auto size = readNum<size_t>(from);
while (size--) {
auto k = read(store, from, Phantom<K> {});
auto v = read(store, from, Phantom<V> {});
auto k = WorkerProto<K>::read(store, from);
auto v = WorkerProto<V>::read(store, from);
resMap.insert_or_assign(std::move(k), std::move(v));
}
return resMap;
}
template<typename K, typename V>
void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
{
out << resMap.size();
for (auto & i : resMap) {
write(store, out, i.first);
write(store, out, i.second);
WorkerProto<K>::write(store, out, i.first);
WorkerProto<V>::write(store, out, i.second);
}
}
}
}