nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-13 22:12:43 +01:00
Code Activity

Merge branch 'read-only-local-store' into overlayfs-store

Browse source
This commit is contained in:
Ben Radford 2023-05-23 09:52:41 +01:00
commit ff12cf3b94
No known key found for this signature in database
GPG key ID: 9DF5D4640AB888D5
91 changed files with 975 additions and 545 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/libexpr/eval.cc
View file

@ -4,6 +4,7 @@
#include "util.hh"
#include "store-api.hh"
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "globals.hh"
#include "eval-inline.hh"
#include "filetransfer.hh"
@ -1058,7 +1059,7 @@ void EvalState::mkOutputString(
? store->printStorePath(*std::move(optOutputPath))
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
: downstreamPlaceholder(*store, drvPath, outputName),
: DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(),
NixStringContext {
NixStringContextElem::Built {
.drvPath = drvPath,
@ -2380,7 +2381,7 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str
// This is testing for the case of CA derivations
auto sExpected = optOutputPath
? store->printStorePath(*optOutputPath)
: downstreamPlaceholder(*store, b.drvPath, output);
: DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render();
if (s != sExpected)
error(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",

4
src/libexpr/eval.hh
View file

@ -483,7 +483,7 @@ public:
* Coerce to `DerivedPath`.
*
* Must be a string which is either a literal store path or a
* "placeholder (see `downstreamPlaceholder()`).
* "placeholder (see `DownstreamPlaceholder`).
*
* Even more importantly, the string context must be exactly one
* element, which is either a `NixStringContextElem::Opaque` or
@ -622,7 +622,7 @@ public:
* @param optOutputPath Optional output path for that string. Must
* be passed if and only if output store object is input-addressed.
* Will be printed to form string if passed, otherwise a placeholder
* will be used (see `downstreamPlaceholder()`).
* will be used (see `DownstreamPlaceholder`).
*/
void mkOutputString(
Value & value,

3
src/libexpr/primops.cc
View file

@ -1,5 +1,6 @@
#include "archive.hh"
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "eval-inline.hh"
#include "eval.hh"
#include "globals.hh"
@ -87,7 +88,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
auto outputs = resolveDerivedPath(*store, drv);
for (auto & [outputName, outputPath] : outputs) {
res.insert_or_assign(
downstreamPlaceholder(*store, drv.drvPath, outputName),
DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(),
store->printStorePath(outputPath)
);
}

9
src/libfetchers/git.cc
View file

@ -62,6 +62,7 @@ std::optional<std::string> readHead(const Path & path)
.program = "git",
// FIXME: use 'HEAD' to avoid returning all refs
.args = {"ls-remote", "--symref", path},
.isInteractive = true,
});
if (status != 0) return std::nullopt;
@ -350,7 +351,7 @@ struct GitInputScheme : InputScheme
args.push_back(destDir);
runProgram("git", true, args);
runProgram("git", true, args, {}, true);
}
std::optional<Path> getSourcePath(const Input & input) override
@ -555,7 +556,7 @@ struct GitInputScheme : InputScheme
: ref == "HEAD"
? *ref
: "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@ -622,7 +623,7 @@ struct GitInputScheme : InputScheme
// everything to ensure we get the rev.
Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
"--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
}
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
@ -649,7 +650,7 @@ struct GitInputScheme : InputScheme
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
}
filter = isNotDotGitDirectory;

4
src/libstore/build/derivation-goal.cc
View file

@ -1152,7 +1152,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Tell the hook all the inputs that have to be copied to the
remote system. */
worker_proto::write(worker.store, hook->sink, inputPaths);
workerProtoWrite(worker.store, hook->sink, inputPaths);
/* Tell the hooks the missing outputs that have to be copied back
from the remote system. */
@ -1163,7 +1163,7 @@ HookReply DerivationGoal::tryBuildHook()
if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
missingOutputs.insert(outputName);
}
worker_proto::write(worker.store, hook->sink, missingOutputs);
workerProtoWrite(worker.store, hook->sink, missingOutputs);
}
hook->sink = FdSink();

2
src/libstore/build/entry-points.cc
View file

@ -110,7 +110,7 @@ void Store::ensurePath(const StorePath & path)
}
void LocalStore::repairPath(const StorePath & path)
void Store::repairPath(const StorePath & path)
{
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);

2
src/libstore/build/local-derivation-goal.cc
View file

@ -1771,6 +1771,8 @@ void LocalDerivationGoal::runChild()
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
if (pathExists(path))
ss.push_back(path);
dirsInChroot.emplace(settings.caFile, "/etc/ssl/certs/ca-certificates.crt");
}
for (auto & i : ss) dirsInChroot.emplace(i, i);

50
src/libstore/daemon.cc
View file

@ -263,7 +263,7 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli
{
std::vector<DerivedPath> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
reqs = WorkerProto<std::vector<DerivedPath>>::read(store, from);
} else {
for (auto & s : readStrings<Strings>(from))
reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
@ -287,7 +287,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryValidPaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, from);
SubstituteFlag substitute = NoSubstitute;
if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
@ -300,7 +300,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
auto res = store->queryValidPaths(paths, substitute);
logger->stopWork();
worker_proto::write(*store, to, res);
workerProtoWrite(*store, to, res);
break;
}
@ -316,11 +316,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, from);
logger->startWork();
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
worker_proto::write(*store, to, res);
workerProtoWrite(*store, to, res);
break;
}
@ -349,7 +349,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
paths = store->queryValidDerivers(path);
else paths = store->queryDerivationOutputs(path);
logger->stopWork();
worker_proto::write(*store, to, paths);
workerProtoWrite(*store, to, paths);
break;
}
@ -367,7 +367,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto outputs = store->queryPartialDerivationOutputMap(path);
logger->stopWork();
worker_proto::write(*store, to, outputs);
workerProtoWrite(*store, to, outputs);
break;
}
@ -393,7 +393,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
auto name = readString(from);
auto camStr = readString(from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto refs = WorkerProto<StorePathSet>::read(*store, from);
bool repairBool;
from >> repairBool;
auto repair = RepairFlag{repairBool};
@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopAddTextToStore: {
std::string suffix = readString(from);
std::string s = readString(from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto refs = WorkerProto<StorePathSet>::read(*store, from);
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
logger->stopWork();
@ -567,7 +567,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto results = store->buildPathsWithResults(drvs, mode);
logger->stopWork();
worker_proto::write(*store, to, results);
workerProtoWrite(*store, to, results);
break;
}
@ -644,7 +644,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(*store, to, builtOutputs);
workerProtoWrite(*store, to, builtOutputs);
}
break;
}
@ -709,7 +709,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopCollectGarbage: {
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
from >> options.ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
@ -779,7 +779,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
else {
to << 1
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
worker_proto::write(*store, to, i->second.references);
workerProtoWrite(*store, to, i->second.references);
to << i->second.downloadSize
<< i->second.narSize;
}
@ -790,11 +790,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
SubstitutablePathInfos infos;
StorePathCAMap pathsMap = {};
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, from);
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
} else
pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
logger->startWork();
store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
@ -802,7 +802,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
for (auto & i : infos) {
to << store->printStorePath(i.first)
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
worker_proto::write(*store, to, i.second.references);
workerProtoWrite(*store, to, i.second.references);
to << i.second.downloadSize << i.second.narSize;
}
break;
@ -812,7 +812,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto paths = store->queryAllValidPaths();
logger->stopWork();
worker_proto::write(*store, to, paths);
workerProtoWrite(*store, to, paths);
break;
}
@ -884,7 +884,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
info.ca = ContentAddress::parseOpt(readString(from));
@ -935,9 +935,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
worker_proto::write(*store, to, willBuild);
worker_proto::write(*store, to, willSubstitute);
worker_proto::write(*store, to, unknown);
workerProtoWrite(*store, to, willBuild);
workerProtoWrite(*store, to, willSubstitute);
workerProtoWrite(*store, to, unknown);
to << downloadSize << narSize;
break;
}
@ -950,7 +950,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
} else {
auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
auto realisation = WorkerProto<Realisation>::read(*store, from);
store->registerDrvOutput(realisation);
}
logger->stopWork();
@ -965,11 +965,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
workerProtoWrite(*store, to, outPaths);
} else {
std::set<Realisation> realisations;
if (info) realisations.insert(*info);
worker_proto::write(*store, to, realisations);
workerProtoWrite(*store, to, realisations);
}
break;
}
@ -1045,7 +1045,7 @@ void processConnection(
auto temp = trusted
? store->isTrustedClient()
: std::optional { NotTrusted };
worker_proto::write(*store, to, temp);
workerProtoWrite(*store, to, temp);
}
/* Send startup error messages to the client. */

15
src/libstore/derivations.cc
View file

@ -1,4 +1,5 @@
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "store-api.hh"
#include "globals.hh"
#include "util.hh"
@ -748,7 +749,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);
@ -796,7 +797,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
},
}, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
workerProtoWrite(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)
@ -810,13 +811,7 @@ std::string hashPlaceholder(const std::string_view outputName)
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
}
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
}
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
@ -880,7 +875,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
for (auto & outputName : inputOutputs) {
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
store.printStorePath(*actualPath));
resolved.inputSrcs.insert(*actualPath);
} else {

12
src/libstore/derivations.hh
View file

@ -6,6 +6,7 @@
#include "hash.hh"
#include "content-address.hh"
#include "repair-flag.hh"
#include "derived-path.hh"
#include "sync.hh"
#include "comparator.hh"
@ -495,17 +496,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
*/
std::string hashPlaceholder(const std::string_view outputName);
/**
* This creates an opaque and almost certainly unique string
* deterministically from a derivation path and output name.
*
* It is used as a placeholder to allow derivations to refer to
* content-addressed paths whose content --- and thus the path
* themselves --- isn't yet known. This occurs when a derivation has a
* dependency which is a CA derivation.
*/
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
extern const Hash impureOutputHash;
}

39
src/libstore/downstream-placeholder.cc Normal file
View file

@ -0,0 +1,39 @@
#include "downstream-placeholder.hh"
#include "derivations.hh"
namespace nix {
std::string DownstreamPlaceholder::render() const
{
return "/" + hash.to_string(Base32, false);
}
DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
const StorePath & drvPath,
std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return DownstreamPlaceholder {
hashString(htSHA256, clearText)
};
}
DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
const DownstreamPlaceholder & placeholder,
std::string_view outputName,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::DynamicDerivations);
auto compressed = compressHash(placeholder.hash, 20);
auto clearText = "nix-computed-output:"
+ compressed.to_string(Base32, false)
+ ":" + std::string { outputName };
return DownstreamPlaceholder {
hashString(htSHA256, clearText)
};
}
}

75
src/libstore/downstream-placeholder.hh Normal file
View file

@ -0,0 +1,75 @@
#pragma once
///@file
#include "hash.hh"
#include "path.hh"
namespace nix {
/**
* Downstream Placeholders are opaque and almost certainly unique values
* used to allow derivations to refer to store objects which are yet to
* be built and for we do not yet have store paths for.
*
* They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`,
* except for the cases involving input addressing or fixed outputs
* where we do know a store path for the derivation output in advance.
*
* Unlike `DerivationPath`, however, `DownstreamPlaceholder` is
* purposefully opaque and obfuscated. This is so they are hard to
* create by accident, and so substituting them (once we know what the
* path to store object is) is unlikely to capture other stuff it
* shouldn't.
*
* We use them with `Derivation`: the `render()` method is called to
* render an opaque string which can be used in the derivation, and the
* resolving logic can substitute those strings for store paths when
* resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`.
*/
class DownstreamPlaceholder
{
/**
* `DownstreamPlaceholder` is just a newtype of `Hash`.
* This its only field.
*/
Hash hash;
/**
* Newtype constructor
*/
DownstreamPlaceholder(Hash hash) : hash(hash) { }
public:
/**
* This creates an opaque and almost certainly unique string
* deterministically from the placeholder.
*/
std::string render() const;
/**
* Create a placeholder for an unknown output of a content-addressed
* derivation.
*
* The derivation itself is known (we have a store path for it), but
* the output doesn't yet have a known store path.
*/
static DownstreamPlaceholder unknownCaOutput(
const StorePath & drvPath,
std::string_view outputName);
/**
* Create a placehold for the output of an unknown derivation.
*
* The derivation is not yet known because it is a dynamic
* derivaiton --- it is itself an output of another derivation ---
* and we just have (another) placeholder for it.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DownstreamPlaceholder unknownDerivation(
const DownstreamPlaceholder & drvPlaceholder,
std::string_view outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
};
}

4
src/libstore/export-import.cc
View file

@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
teeSink
<< exportMagic
<< printStorePath(path);
worker_proto::write(*this, teeSink, info->references);
workerProtoWrite(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
//Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
auto references = WorkerProto<StorePathSet>::read(*this, source);
auto deriver = readString(source);
auto narHash = hashString(htSHA256, saved.s);

5
src/libstore/gc.cc
View file

@ -110,6 +110,11 @@ void LocalStore::createTempRootsFile()
void LocalStore::addTempRoot(const StorePath & path)
{
if (readOnly) {
debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways.");
return;
}
createTempRootsFile();
/* Open/create the global GC lock file. */

27
src/libstore/legacy-ssh-store.cc
View file

@ -146,7 +146,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
auto deriver = readString(conn->from);
if (deriver != "")
info->deriver = parseStorePath(deriver);
info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
readLongLong(conn->from); // download size
info->narSize = readLongLong(conn->from);
@ -180,7 +180,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
worker_proto::write(*this, conn->to, info.references);
workerProtoWrite(*this, conn->to, info.references);
conn->to
<< info.registrationTime
<< info.narSize
@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
conn->to
<< exportMagic
<< printStorePath(info.path);
worker_proto::write(*this, conn->to, info.references);
workerProtoWrite(*this, conn->to, info.references);
conn->to
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0
@ -294,7 +294,7 @@ public:
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
for (auto && [output, realisation] : builtOutputs)
status.builtOutputs.insert_or_assign(
std::move(output.outputName),
@ -344,6 +344,17 @@ public:
virtual ref<FSAccessor> getFSAccessor() override
{ unsupported("getFSAccessor"); }
/**
* The default instance would schedule the work on the client side, but
* for consistency with `buildPaths` and `buildDerivation` it should happen
* on the remote side.
*
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override
@ -358,10 +369,10 @@ public:
conn->to
<< cmdQueryClosure
<< includeOutputs;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
conn->to.flush();
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
out.insert(i);
}
@ -374,10 +385,10 @@ public:
<< cmdQueryValidPaths
<< false // lock
<< maybeSubstitute;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
conn->to.flush();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
void connect() override

43
src/libstore/local-store.cc
View file

@ -202,10 +202,16 @@ LocalStore::LocalStore(const Params & params)
createSymlink(profilesDir, gcRootsDir + "/profiles");
}
if (readOnly) {
experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore);
}
for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
createDirs(perUserDir);
if (chmod(perUserDir.c_str(), 0755) == -1)
throw SysError("could not set permissions on '%s' to 755", perUserDir);
if (!readOnly) {
if (chmod(perUserDir.c_str(), 0755) == -1)
throw SysError("could not set permissions on '%s' to 755", perUserDir);
}
}
/* Optionally, create directories and set permissions for a
@ -269,10 +275,12 @@ LocalStore::LocalStore(const Params & params)
/* Acquire the big fat lock in shared mode to make sure that no
schema upgrade is in progress. */
Path globalLockPath = dbDir + "/big-lock";
globalLock = openLockFile(globalLockPath.c_str(), true);
if (!readOnly) {
Path globalLockPath = dbDir + "/big-lock";
globalLock = openLockFile(globalLockPath.c_str(), true);
}
if (!lockFile(globalLock.get(), ltRead, false)) {
if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) {
printInfo("waiting for the big Nix store lock...");
lockFile(globalLock.get(), ltRead, true);
}
@ -280,6 +288,14 @@ LocalStore::LocalStore(const Params & params)
/* Check the current database schema and if necessary do an
upgrade. */
int curSchema = getSchema();
if (readOnly && curSchema < nixSchemaVersion) {
debug("current schema version: %d", curSchema);
debug("supported schema version: %d", nixSchemaVersion);
throw Error(curSchema == 0 ?
"database does not exist, and cannot be created in read-only mode" :
"database schema needs migrating, but this cannot be done in read-only mode");
}
if (curSchema > nixSchemaVersion)
throw Error("current Nix store schema is version %1%, but I only support %2%",
curSchema, nixSchemaVersion);
@ -344,7 +360,11 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false);
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
if (!readOnly) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
} else {
throw Error("need to migrate to CA schema, but this cannot be done in read-only mode");
}
}
/* Prepare SQL statements. */
@ -475,13 +495,20 @@ int LocalStore::getSchema()
void LocalStore::openDB(State & state, bool create)
{
if (access(dbDir.c_str(), R_OK | W_OK))
if (create && readOnly) {
throw Error("unable to create database while in read-only mode");
}
if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK)))
throw SysError("Nix database directory '%1%' is not writable", dbDir);
/* Open the Nix database. */
std::string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
state.db = SQLite(dbPath, create);
auto openMode = readOnly ? SQLiteOpenMode::Immutable
: create ? SQLiteOpenMode::Normal
: SQLiteOpenMode::NoCreate;
state.db = SQLite(dbPath, openMode);
#ifdef __CYGWIN__
/* The cygwin version of sqlite3 has a patch which calls

18
src/libstore/local-store.hh
View file

@ -46,6 +46,22 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
"require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
Setting<bool> readOnly{(StoreConfig*) this,
false,
"read-only",
R"(
Allow this store to be opened when its database is on a read-only filesystem.
Normally Nix will attempt to open the store database in read-write mode, even
for querying (when write access is not needed). This causes it to fail if the
database is on a read-only filesystem.
Enable read-only mode to disable locking and open the SQLite database with the
**immutable** parameter set. Do not use this unless the filesystem is read-only.
Using it when the filesystem is writable can cause incorrect query results or
corruption errors if the database is changed by another process.
)"};
const std::string name() override { return "Local Store"; }
std::string doc() override;
@ -240,8 +256,6 @@ public:
void vacuumDB();
void repairPath(const StorePath & path) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
/**

5
src/libstore/path-info.cc
View file

@ -1,5 +1,6 @@
#include "path-info.hh"
#include "worker-protocol.hh"
#include "store-api.hh"
namespace nix {
@ -131,7 +132,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
auto narHash = Hash::parseAny(readString(source), htSHA256);
ValidPathInfo info(path, narHash);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
info.references = worker_proto::read(store, source, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(store, source);
source >> info.registrationTime >> info.narSize;
if (format >= 16) {
source >> info.ultimate;
@ -152,7 +153,7 @@ void ValidPathInfo::write(
sink << store.printStorePath(path);
sink << (deriver ? store.printStorePath(*deriver) : "")
<< narHash.to_string(Base16, false);
worker_proto::write(store, sink, references);
workerProtoWrite(store, sink, references);
sink << registrationTime << narSize;
if (format >= 16) {
sink << ultimate

245
src/libstore/remote-store.cc
View file

@ -18,189 +18,6 @@
namespace nix {
namespace worker_proto {
std::string read(const Store & store, Source & from, Phantom<std::string> _)
{
return readString(from);
}
void write(const Store & store, Sink & out, const std::string & str)
{
out << str;
}
StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
{
return store.parseStorePath(readString(from));
}
void write(const Store & store, Sink & out, const StorePath & storePath)
{
out << store.printStorePath(storePath);
}
std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _)
{
auto temp = readNum<uint8_t>(from);
switch (temp) {
case 0:
return std::nullopt;
case 1:
return { Trusted };
case 2:
return { NotTrusted };
default:
throw Error("Invalid trusted status from remote");
}
}
void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
{
if (!optTrusted)
out << (uint8_t)0;
else {
switch (*optTrusted) {
case Trusted:
out << (uint8_t)1;
break;
case NotTrusted:
out << (uint8_t)2;
break;
default:
assert(false);
};
}
}
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
{
return ContentAddress::parse(readString(from));
}
void write(const Store & store, Sink & out, const ContentAddress & ca)
{
out << renderContentAddress(ca);
}
DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
{
auto s = readString(from);
return DerivedPath::parseLegacy(store, s);
}
void write(const Store & store, Sink & out, const DerivedPath & req)
{
out << req.to_string_legacy(store);
}
Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
{
std::string rawInput = readString(from);
return Realisation::fromJSON(
nlohmann::json::parse(rawInput),
"remote-protocol"
);
}
void write(const Store & store, Sink & out, const Realisation & realisation)
{
out << realisation.toJSON().dump();
}
DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
{
return DrvOutput::parse(readString(from));
}
void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
{
out << drvOutput.to_string();
}
KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _)
{
auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
auto br = worker_proto::read(store, from, Phantom<BuildResult> {});
return KeyedBuildResult {
std::move(br),
/* .path = */ std::move(path),
};
}
void write(const Store & store, Sink & to, const KeyedBuildResult & res)
{
worker_proto::write(store, to, res.path);
worker_proto::write(store, to, static_cast<const BuildResult &>(res));
}
BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
{
BuildResult res;
res.status = (BuildResult::Status) readInt(from);
from
>> res.errorMsg
>> res.timesBuilt
>> res.isNonDeterministic
>> res.startTime
>> res.stopTime;
auto builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
return res;
}
void write(const Store & store, Sink & to, const BuildResult & res)
{
to
<< res.status
<< res.errorMsg
<< res.timesBuilt
<< res.isNonDeterministic
<< res.startTime
<< res.stopTime;
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(store, to, builtOutputs);
}
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{
auto s = readString(from);
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
}
void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
{
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
}
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
{
return ContentAddress::parseOpt(readString(from));
}
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
{
out << (caOpt ? renderContentAddress(*caOpt) : "");
}
}
/* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
: RemoteStoreConfig(params)
@ -283,7 +100,7 @@ void RemoteStore::initConnection(Connection & conn)
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {});
conn.remoteTrustsUs = WorkerProto<std::optional<TrustedFlag>>::read(*this, conn.from);
} else {
// We don't know the answer; protocol to old.
conn.remoteTrustsUs = std::nullopt;
@ -410,12 +227,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
return res;
} else {
conn->to << wopQueryValidPaths;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
}
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
}
@ -425,7 +242,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
auto conn(getConnection());
conn->to << wopQueryAllValidPaths;
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
@ -442,9 +259,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
return res;
} else {
conn->to << wopQuerySubstitutablePaths;
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
}
@ -466,7 +283,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
infos.insert_or_assign(i.first, std::move(info));
@ -479,9 +296,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
StorePathSet paths;
for (auto & path : pathsMap)
paths.insert(path.first);
worker_proto::write(*this, conn->to, paths);
workerProtoWrite(*this, conn->to, paths);
} else
worker_proto::write(*this, conn->to, pathsMap);
workerProtoWrite(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
@ -489,7 +306,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
}
@ -532,7 +349,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
auto conn(getConnection());
conn->to << wopQueryReferrers << printStorePath(path);
conn.processStderr();
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
referrers.insert(i);
}
@ -542,7 +359,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
auto conn(getConnection());
conn->to << wopQueryValidDerivers << printStorePath(path);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
@ -554,7 +371,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
auto conn(getConnection());
conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
return WorkerProto<StorePathSet>::read(*this, conn->from);
}
@ -564,7 +381,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat
auto conn(getConnection());
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
return WorkerProto<std::map<std::string, std::optional<StorePath>>>::read(*this, conn->from);
} else {
// Fallback for old daemon versions.
// For floating-CA derivations (and their co-dependencies) this is an
@ -610,7 +427,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
<< wopAddToStore
<< name
<< caMethod.render(hashType);
worker_proto::write(*this, conn->to, references);
workerProtoWrite(*this, conn->to, references);
conn->to << repair;
// The dump source may invoke the store, so we need to make some room.
@ -635,7 +452,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
name, printHashType(hashType));
std::string s = dump.drain();
conn->to << wopAddTextToStore << name << s;
worker_proto::write(*this, conn->to, references);
workerProtoWrite(*this, conn->to, references);
conn.processStderr();
},
[&](const FileIngestionMethod & fim) -> void {
@ -701,7 +518,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
sink
<< exportMagic
<< printStorePath(info.path);
worker_proto::write(*this, sink, info.references);
workerProtoWrite(*this, sink, info.references);
sink
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0 // == no legacy signature
@ -711,7 +528,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn.processStderr(0, source2.get());
auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
auto importedPaths = WorkerProto<StorePathSet>::read(*this, conn->from);
assert(importedPaths.size() <= 1);
}
@ -720,7 +537,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
worker_proto::write(*this, conn->to, info.references);
workerProtoWrite(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
@ -793,7 +610,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
conn->to << info.id.to_string();
conn->to << std::string(info.outPath.to_string());
} else {
worker_proto::write(*this, conn->to, info);
workerProtoWrite(*this, conn->to, info);
}
conn.processStderr();
}
@ -815,14 +632,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
auto real = [&]() -> std::shared_ptr<const Realisation> {
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
auto outPaths = worker_proto::read(
*this, conn->from, Phantom<std::set<StorePath>> {});
auto outPaths = WorkerProto<std::set<StorePath>>::read(
*this, conn->from);
if (outPaths.empty())
return nullptr;
return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
} else {
auto realisations = worker_proto::read(
*this, conn->from, Phantom<std::set<Realisation>> {});
auto realisations = WorkerProto<std::set<Realisation>>::read(
*this, conn->from);
if (realisations.empty())
return nullptr;
return std::make_shared<const Realisation>(*realisations.begin());
@ -836,7 +653,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
{
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
worker_proto::write(store, conn->to, reqs);
workerProtoWrite(store, conn->to, reqs);
} else {
Strings ss;
for (auto & p : reqs) {
@ -906,7 +723,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
writeDerivedPaths(*this, conn, paths);
conn->to << buildMode;
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<std::vector<KeyedBuildResult>> {});
return WorkerProto<std::vector<KeyedBuildResult>>::read(*this, conn->from);
} else {
// Avoid deadlock.
conn_.reset();
@ -989,7 +806,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
auto builtOutputs = WorkerProto<DrvOutputs>::read(*this, conn->from);
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
@ -1048,7 +865,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
conn->to
<< wopCollectGarbage << options.action;
worker_proto::write(*this, conn->to, options.pathsToDelete);
workerProtoWrite(*this, conn->to, options.pathsToDelete);
conn->to << options.ignoreLiveness
<< options.maxFreed
/* removed options */
@ -1107,9 +924,9 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
conn->to << wopQueryMissing;
writeDerivedPaths(*this, conn, targets);
conn.processStderr();
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
willBuild = WorkerProto<StorePathSet>::read(*this, conn->from);
willSubstitute = WorkerProto<StorePathSet>::read(*this, conn->from);
unknown = WorkerProto<StorePathSet>::read(*this, conn->from);
conn->from >> downloadSize >> narSize;
return;
}

11
src/libstore/remote-store.hh
View file

@ -137,6 +137,17 @@ public:
bool verifyStore(bool checkContents, RepairFlag repair) override;
/**
* The default instance would schedule the work on the client side, but
* for consistency with `buildPaths` and `buildDerivation` it should happen
* on the remote side.
*
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
void queryMissing(const std::vector<DerivedPath> & targets,

11
src/libstore/sqlite.cc
View file

@ -1,6 +1,7 @@
#include "sqlite.hh"
#include "globals.hh"
#include "util.hh"
#include "url.hh"
#include <sqlite3.h>
@ -50,15 +51,17 @@ static void traceSQL(void * x, const char * sql)
notice("SQL<[%1%]>", sql);
};
SQLite::SQLite(const Path & path, bool create)
SQLite::SQLite(const Path & path, SQLiteOpenMode mode)
{
// useSQLiteWAL also indicates what virtual file system we need. Using
// `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
// for Linux (WSL) where useSQLiteWAL should be false by default.
const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
int flags = SQLITE_OPEN_READWRITE;
if (create) flags |= SQLITE_OPEN_CREATE;
int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs);
bool immutable = mode == SQLiteOpenMode::Immutable;
int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE;
if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE;
auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0");
int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs);
if (ret != SQLITE_OK) {
const char * err = sqlite3_errstr(ret);
throw Error("cannot open SQLite database '%s': %s", path, err);

23
src/libstore/sqlite.hh
View file

@ -11,6 +11,27 @@ struct sqlite3_stmt;
namespace nix {
enum class SQLiteOpenMode {
/**
* Open the database in read-write mode.
* If the database does not exist, it will be created.
*/
Normal,
/**
* Open the database in read-write mode.
* Fails with an error if the database does not exist.
*/
NoCreate,
/**
* Open the database in immutable mode.
* In addition to the database being read-only,
* no wal or journal files will be created by sqlite.
* Use this mode if the database is on a read-only filesystem.
* Fails with an error if the database does not exist.
*/
Immutable
};
/**
* RAII wrapper to close a SQLite database automatically.
*/
@ -18,7 +39,7 @@ struct SQLite
{
sqlite3 * db = 0;
SQLite() { }
SQLite(const Path & path, bool create = true);
SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal);
SQLite(const SQLite & from) = delete;
SQLite& operator = (const SQLite & from) = delete;
SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }

23
src/libstore/ssh.cc
View file

@ -41,6 +41,11 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
args.push_back("-oLocalCommand=echo started");
}
bool SSHMaster::isMasterRunning() {
auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true});
return res.first == 0;
}
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
{
Path socketPath = startMaster();
@ -97,7 +102,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
// Wait for the SSH connection to be established,
// So that we don't overwrite the password prompt with our progress bar.
if (!fakeSSH && !useMaster) {
if (!fakeSSH && !useMaster && !isMasterRunning()) {
std::string reply;
try {
reply = readLine(out.readSide.get());
@ -133,6 +138,8 @@ Path SSHMaster::startMaster()
logger->pause();
Finally cleanup = [&]() { logger->resume(); };
bool wasMasterRunning = isMasterRunning();
state->sshMaster = startProcess([&]() {
restoreProcessContext();
@ -152,13 +159,15 @@ Path SSHMaster::startMaster()
out.writeSide = -1;
std::string reply;
try {
reply = readLine(out.readSide.get());
} catch (EndOfFile & e) { }
if (!wasMasterRunning) {
std::string reply;
try {
reply = readLine(out.readSide.get());
} catch (EndOfFile & e) { }
if (reply != "started")
throw Error("failed to start SSH master connection to '%s'", host);
if (reply != "started")
throw Error("failed to start SSH master connection to '%s'", host);
}
return state->socketPath;
}

1
src/libstore/ssh.hh
View file

@ -28,6 +28,7 @@ private:
Sync<State> state_;
void addCommonSSHOpts(Strings & args);
bool isMasterRunning();
public:

3
src/libstore/store-api.hh
View file

@ -679,8 +679,7 @@ public:
* Repair the contents of the given path by redownloading it using
* a substituter (if available).
*/
virtual void repairPath(const StorePath & path)
{ unsupported("repairPath"); }
virtual void repairPath(const StorePath & path);
/**
* Add signatures to the specified store path. The signatures are

33
src/libstore/tests/downstream-placeholder.cc Normal file
View file

@ -0,0 +1,33 @@
#include <gtest/gtest.h>
#include "downstream-placeholder.hh"
namespace nix {
TEST(DownstreamPlaceholder, unknownCaOutput) {
ASSERT_EQ(
DownstreamPlaceholder::unknownCaOutput(
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
"out").render(),
"/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz");
}
TEST(DownstreamPlaceholder, unknownDerivation) {
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
ASSERT_EQ(
DownstreamPlaceholder::unknownDerivation(
DownstreamPlaceholder::unknownCaOutput(
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" },
"out"),
"out",
mockXpSettings).render(),
"/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w");
}
}

192
src/libstore/worker-protocol.cc Normal file
View file

@ -0,0 +1,192 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "build-result.hh"
#include "worker-protocol.hh"
#include "archive.hh"
#include "derivations.hh"
#include <nlohmann/json.hpp>
namespace nix {
std::string WorkerProto<std::string>::read(const Store & store, Source & from)
{
return readString(from);
}
void WorkerProto<std::string>::write(const Store & store, Sink & out, const std::string & str)
{
out << str;
}
StorePath WorkerProto<StorePath>::read(const Store & store, Source & from)
{
return store.parseStorePath(readString(from));
}
void WorkerProto<StorePath>::write(const Store & store, Sink & out, const StorePath & storePath)
{
out << store.printStorePath(storePath);
}
std::optional<TrustedFlag> WorkerProto<std::optional<TrustedFlag>>::read(const Store & store, Source & from)
{
auto temp = readNum<uint8_t>(from);
switch (temp) {
case 0:
return std::nullopt;
case 1:
return { Trusted };
case 2:
return { NotTrusted };
default:
throw Error("Invalid trusted status from remote");
}
}
void WorkerProto<std::optional<TrustedFlag>>::write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted)
{
if (!optTrusted)
out << (uint8_t)0;
else {
switch (*optTrusted) {
case Trusted:
out << (uint8_t)1;
break;
case NotTrusted:
out << (uint8_t)2;
break;
default:
assert(false);
};
}
}
ContentAddress WorkerProto<ContentAddress>::read(const Store & store, Source & from)
{
return ContentAddress::parse(readString(from));
}
void WorkerProto<ContentAddress>::write(const Store & store, Sink & out, const ContentAddress & ca)
{
out << renderContentAddress(ca);
}
DerivedPath WorkerProto<DerivedPath>::read(const Store & store, Source & from)
{
auto s = readString(from);
return DerivedPath::parseLegacy(store, s);
}
void WorkerProto<DerivedPath>::write(const Store & store, Sink & out, const DerivedPath & req)
{
out << req.to_string_legacy(store);
}
Realisation WorkerProto<Realisation>::read(const Store & store, Source & from)
{
std::string rawInput = readString(from);
return Realisation::fromJSON(
nlohmann::json::parse(rawInput),
"remote-protocol"
);
}
void WorkerProto<Realisation>::write(const Store & store, Sink & out, const Realisation & realisation)
{
out << realisation.toJSON().dump();
}
DrvOutput WorkerProto<DrvOutput>::read(const Store & store, Source & from)
{
return DrvOutput::parse(readString(from));
}
void WorkerProto<DrvOutput>::write(const Store & store, Sink & out, const DrvOutput & drvOutput)
{
out << drvOutput.to_string();
}
KeyedBuildResult WorkerProto<KeyedBuildResult>::read(const Store & store, Source & from)
{
auto path = WorkerProto<DerivedPath>::read(store, from);
auto br = WorkerProto<BuildResult>::read(store, from);
return KeyedBuildResult {
std::move(br),
/* .path = */ std::move(path),
};
}
void WorkerProto<KeyedBuildResult>::write(const Store & store, Sink & to, const KeyedBuildResult & res)
{
workerProtoWrite(store, to, res.path);
workerProtoWrite(store, to, static_cast<const BuildResult &>(res));
}
BuildResult WorkerProto<BuildResult>::read(const Store & store, Source & from)
{
BuildResult res;
res.status = (BuildResult::Status) readInt(from);
from
>> res.errorMsg
>> res.timesBuilt
>> res.isNonDeterministic
>> res.startTime
>> res.stopTime;
auto builtOutputs = WorkerProto<DrvOutputs>::read(store, from);
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
return res;
}
void WorkerProto<BuildResult>::write(const Store & store, Sink & to, const BuildResult & res)
{
to
<< res.status
<< res.errorMsg
<< res.timesBuilt
<< res.isNonDeterministic
<< res.startTime
<< res.stopTime;
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
workerProtoWrite(store, to, builtOutputs);
}
std::optional<StorePath> WorkerProto<std::optional<StorePath>>::read(const Store & store, Source & from)
{
auto s = readString(from);
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
}
void WorkerProto<std::optional<StorePath>>::write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
{
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
}
std::optional<ContentAddress> WorkerProto<std::optional<ContentAddress>>::read(const Store & store, Source & from)
{
return ContentAddress::parseOpt(readString(from));
}
void WorkerProto<std::optional<ContentAddress>>::write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
{
out << (caOpt ? renderContentAddress(*caOpt) : "");
}
}

123
src/libstore/worker-protocol.hh
View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "store-api.hh"
#include "serialise.hh"
namespace nix {
@ -79,41 +78,81 @@ typedef enum {
class Store;
struct Source;
// items being serialized
struct DerivedPath;
struct DrvOutput;
struct Realisation;
struct BuildResult;
struct KeyedBuildResult;
enum TrustedFlag : bool;
/**
* Used to guide overloading
* Data type for canonical pairs of serializers for the worker protocol.
*
* See https://en.cppreference.com/w/cpp/language/adl for the broader
* concept of what is going on here.
*/
template<typename T>
struct Phantom {};
struct WorkerProto {
static T read(const Store & store, Source & from);
static void write(const Store & store, Sink & out, const T & t);
};
/**
* Wrapper function around `WorkerProto<T>::write` that allows us to
* infer the type instead of having to write it down explicitly.
*/
template<typename T>
void workerProtoWrite(const Store & store, Sink & out, const T & t)
{
WorkerProto<T>::write(store, out, t);
}
namespace worker_proto {
/* FIXME maybe move more stuff inside here */
/**
* Declare a canonical serializer pair for the worker protocol.
*
* We specialize the struct merely to indicate that we are implementing
* the function for the given type.
*
* Some sort of `template<...>` must be used with the caller for this to
* be legal specialization syntax. See below for what that looks like in
* practice.
*/
#define MAKE_WORKER_PROTO(T) \
struct WorkerProto< T > { \
static T read(const Store & store, Source & from); \
static void write(const Store & store, Sink & out, const T & t); \
};
#define MAKE_WORKER_PROTO(TEMPLATE, T) \
TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
TEMPLATE void write(const Store & store, Sink & out, const T & str)
template<>
MAKE_WORKER_PROTO(std::string);
template<>
MAKE_WORKER_PROTO(StorePath);
template<>
MAKE_WORKER_PROTO(ContentAddress);
template<>
MAKE_WORKER_PROTO(DerivedPath);
template<>
MAKE_WORKER_PROTO(Realisation);
template<>
MAKE_WORKER_PROTO(DrvOutput);
template<>
MAKE_WORKER_PROTO(BuildResult);
template<>
MAKE_WORKER_PROTO(KeyedBuildResult);
template<>
MAKE_WORKER_PROTO(std::optional<TrustedFlag>);
MAKE_WORKER_PROTO(, std::string);
MAKE_WORKER_PROTO(, StorePath);
MAKE_WORKER_PROTO(, ContentAddress);
MAKE_WORKER_PROTO(, DerivedPath);
MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(, BuildResult);
MAKE_WORKER_PROTO(, KeyedBuildResult);
MAKE_WORKER_PROTO(, std::optional<TrustedFlag>);
template<typename T>
MAKE_WORKER_PROTO(std::vector<T>);
template<typename T>
MAKE_WORKER_PROTO(std::set<T>);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
#define X_ template<typename K, typename V>
#define Y_ std::map<K, V>
MAKE_WORKER_PROTO(X_, Y_);
template<typename K, typename V>
#define X_ std::map<K, V>
MAKE_WORKER_PROTO(X_);
#undef X_
#undef Y_
/**
* These use the empty string for the null case, relying on the fact
@ -129,72 +168,72 @@ MAKE_WORKER_PROTO(X_, Y_);
* worker protocol harder to implement in other languages where such
* specializations may not be allowed.
*/
MAKE_WORKER_PROTO(, std::optional<StorePath>);
MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
template<>
MAKE_WORKER_PROTO(std::optional<StorePath>);
template<>
MAKE_WORKER_PROTO(std::optional<ContentAddress>);
template<typename T>
std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
std::vector<T> WorkerProto<std::vector<T>>::read(const Store & store, Source & from)
{
std::vector<T> resSet;
auto size = readNum<size_t>(from);
while (size--) {
resSet.push_back(read(store, from, Phantom<T> {}));
resSet.push_back(WorkerProto<T>::read(store, from));
}
return resSet;
}
template<typename T>
void write(const Store & store, Sink & out, const std::vector<T> & resSet)
void WorkerProto<std::vector<T>>::write(const Store & store, Sink & out, const std::vector<T> & resSet)
{
out << resSet.size();
for (auto & key : resSet) {
write(store, out, key);
WorkerProto<T>::write(store, out, key);
}
}
template<typename T>
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
std::set<T> WorkerProto<std::set<T>>::read(const Store & store, Source & from)
{
std::set<T> resSet;
auto size = readNum<size_t>(from);
while (size--) {
resSet.insert(read(store, from, Phantom<T> {}));
resSet.insert(WorkerProto<T>::read(store, from));
}
return resSet;
}
template<typename T>
void write(const Store & store, Sink & out, const std::set<T> & resSet)
void WorkerProto<std::set<T>>::write(const Store & store, Sink & out, const std::set<T> & resSet)
{
out << resSet.size();
for (auto & key : resSet) {
write(store, out, key);
WorkerProto<T>::write(store, out, key);
}
}
template<typename K, typename V>
std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
std::map<K, V> WorkerProto<std::map<K, V>>::read(const Store & store, Source & from)
{
std::map<K, V> resMap;
auto size = readNum<size_t>(from);
while (size--) {
auto k = read(store, from, Phantom<K> {});
auto v = read(store, from, Phantom<V> {});
auto k = WorkerProto<K>::read(store, from);
auto v = WorkerProto<V>::read(store, from);
resMap.insert_or_assign(std::move(k), std::move(v));
}
return resMap;
}
template<typename K, typename V>
void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
void WorkerProto<std::map<K, V>>::write(const Store & store, Sink & out, const std::map<K, V> & resMap)
{
out << resMap.size();
for (auto & i : resMap) {
write(store, out, i.first);
write(store, out, i.second);
WorkerProto<K>::write(store, out, i.first);
WorkerProto<V>::write(store, out, i.second);
}
}
}
}

21
src/libutil/experimental-features.cc
View file

@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
std::string_view description;
};
constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{
{
.tag = Xp::CaDerivations,
.name = "ca-derivations",
@ -207,6 +207,25 @@ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
- "text hashing" derivation outputs, so we can build .drv
files.
- dependencies in derivations on the outputs of
derivations that are themselves derivations outputs.
)",
},
{
.tag = Xp::ReadOnlyLocalStore,
.name = "read-only-local-store",
.description = R"(
Allow the use of the `read-only` parameter in local store URIs.
Set this parameter to `true` to allow stores with databases on read-only
filesystems to be opened for querying; ordinarily Nix will refuse to do this.
Enabling this setting disables the locking required for safe concurrent
access, so you should be certain that the database will not be changed.
While the filesystem the database resides on might be read-only to this
process, consider whether another user, process, or system, might have
write access to it.
)",
},
}};

1
src/libutil/experimental-features.hh
View file

@ -30,6 +30,7 @@ enum struct ExperimentalFeature
DiscardReferences,
DaemonTrustOverride,
DynamicDerivations,
ReadOnlyLocalStore,
};
/**

14
src/libutil/util.cc
View file

@ -1141,9 +1141,9 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
}
std::string runProgram(Path program, bool searchPath, const Strings & args,
const std::optional<std::string> & input)
const std::optional<std::string> & input, bool isInteractive)
{
auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive});
if (!statusOk(res.first))
throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
@ -1193,6 +1193,16 @@ void runProgram2(const RunOptions & options)
// case), so we can't use it if we alter the environment
processOptions.allowVfork = !options.environment;
std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
if (options.isInteractive) {
logger->pause();
resumeLoggerDefer.emplace(
[]() {
logger->resume();
}
);
}
/* Fork. */
Pid pid = startProcess([&]() {
if (options.environment)

3
src/libutil/util.hh
View file

@ -415,7 +415,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
*/
std::string runProgram(Path program, bool searchPath = false,
const Strings & args = Strings(),
const std::optional<std::string> & input = {});
const std::optional<std::string> & input = {}, bool isInteractive = false);
struct RunOptions
{
@ -430,6 +430,7 @@ struct RunOptions
Source * standardIn = nullptr;
Sink * standardOut = nullptr;
bool mergeStderrToStdout = false;
bool isInteractive = false;
};
std::pair<int, std::string> runProgram(RunOptions && options);

7
src/nix-collect-garbage/nix-collect-garbage.cc
View file

@ -77,7 +77,12 @@ static int main_nix_collect_garbage(int argc, char * * argv)
return true;
});
if (removeOld) removeOldGenerations(profilesDir());
if (removeOld) {
std::set<Path> dirsToClean = {
profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())};
for (auto & dir : dirsToClean)
removeOldGenerations(dir);
}
// Run the actual garbage collector.
if (!dryRun) {

18
src/nix-store/nix-store.cc
View file

@ -849,7 +849,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdQueryValidPaths: {
bool lock = readInt(in);
bool substitute = readInt(in);
auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, in);
if (lock && writeAllowed)
for (auto & path : paths)
store->addTempRoot(path);
@ -858,19 +858,19 @@ static void opServe(Strings opFlags, Strings opArgs)
store->substitutePaths(paths);
}
worker_proto::write(*store, out, store->queryValidPaths(paths));
workerProtoWrite(*store, out, store->queryValidPaths(paths));
break;
}
case cmdQueryPathInfos: {
auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {});
auto paths = WorkerProto<StorePathSet>::read(*store, in);
// !!! Maybe we want a queryPathInfos?
for (auto & i : paths) {
try {
auto info = store->queryPathInfo(i);
out << store->printStorePath(info->path)
<< (info->deriver ? store->printStorePath(*info->deriver) : "");
worker_proto::write(*store, out, info->references);
workerProtoWrite(*store, out, info->references);
// !!! Maybe we want compression?
out << info->narSize // downloadSize
<< info->narSize;
@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdExportPaths: {
readInt(in); // obsolete
store->exportPaths(worker_proto::read(*store, in, Phantom<StorePathSet> {}), out);
store->exportPaths(WorkerProto<StorePathSet>::read(*store, in), out);
break;
}
@ -944,7 +944,7 @@ static void opServe(Strings opFlags, Strings opArgs)
DrvOutputs builtOutputs;
for (auto & [output, realisation] : status.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
worker_proto::write(*store, out, builtOutputs);
workerProtoWrite(*store, out, builtOutputs);
}
break;
@ -953,9 +953,9 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdQueryClosure: {
bool includeOutputs = readInt(in);
StorePathSet closure;
store->computeFSClosure(worker_proto::read(*store, in, Phantom<StorePathSet> {}),
store->computeFSClosure(WorkerProto<StorePathSet>::read(*store, in),
closure, false, includeOutputs);
worker_proto::write(*store, out, closure);
workerProtoWrite(*store, out, closure);
break;
}
@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs)
};
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
info.references = WorkerProto<StorePathSet>::read(*store, in);
in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in);
info.ca = ContentAddress::parseOpt(readString(in));

3
src/nix/app.cc
View file

@ -7,6 +7,7 @@
#include "names.hh"
#include "command.hh"
#include "derivations.hh"
#include "downstream-placeholder.hh"
namespace nix {
@ -23,7 +24,7 @@ StringPairs resolveRewrites(
if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
for (auto & [ outputName, outputPath ] : drvDep->outputs)
res.emplace(
downstreamPlaceholder(store, drvDep->drvPath, outputName),
DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(),
store.printStorePath(outputPath)
);
return res;

2
src/nix/build.md
View file

@ -44,7 +44,7 @@ R""(
`release.nix`:
```console
# nix build -f release.nix build.x86_64-linux
# nix build --file release.nix build.x86_64-linux
```
* Build a NixOS system configuration from a flake, and make a profile

2
src/nix/copy.md
View file

@ -15,7 +15,7 @@ R""(
SSH:
```console
# nix copy -s --to ssh://server /run/current-system
# nix copy --substitute-on-destination --to ssh://server /run/current-system
```
The `-s` flag causes the remote machine to try to substitute missing

2
src/nix/develop.md
View file

@ -69,7 +69,7 @@ R""(
* Run a series of script commands:
```console
# nix develop --command bash -c "mkdir build && cmake .. && make"
# nix develop --command bash --command "mkdir build && cmake .. && make"
```
# Description

2
src/nix/eval.md
View file

@ -18,7 +18,7 @@ R""(
* Evaluate a Nix expression from a file:
```console
# nix eval -f ./my-nixpkgs hello.name
# nix eval --file ./my-nixpkgs hello.name
```
* Get the current version of the `nixpkgs` flake:

2
src/nix/flake-check.md
View file

@ -68,6 +68,6 @@ The following flake output attributes must be
In addition, the `hydraJobs` output is evaluated in the same way as
Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested
attribute set of derivations). Similarly, the
`legacyPackages`.*system* output is evaluated like `nix-env -qa`.
`legacyPackages`.*system* output is evaluated like `nix-env --query --available `.
)""

4
src/nix/nar-ls.md
View file

@ -5,7 +5,7 @@ R""(
* To list a specific file in a NAR:
```console
# nix nar ls -l ./hello.nar /bin/hello
# nix nar ls --long ./hello.nar /bin/hello
-r-xr-xr-x 38184 hello
```
@ -13,7 +13,7 @@ R""(
format:
```console
# nix nar ls --json -R ./hello.nar /bin
# nix nar ls --json --recursive ./hello.nar /bin
{"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}}
```

4
src/nix/nix.md
View file

@ -197,7 +197,7 @@ operate are determined as follows:
of all outputs of the `glibc` package in the binary cache:
```console
# nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
# nix path-info --closure-size --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
/nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200
/nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896
/nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312
@ -208,7 +208,7 @@ operate are determined as follows:
and likewise, using a store path to a "drv" file to specify the derivation:
```console
# nix path-info -S '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*'
# nix path-info --closure-size '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*'
…
```
* If you didn't specify the desired outputs, but the derivation has an

8
src/nix/path-info.md
View file

@ -13,7 +13,7 @@ R""(
closure, sorted by size:
```console
# nix path-info -rS /run/current-system | sort -nk2
# nix path-info --recursive --closure-size /run/current-system | sort -nk2
/nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96
/nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112
…
@ -25,7 +25,7 @@ R""(
readable sizes:
```console
# nix path-info -rsSh nixpkgs#rustc
# nix path-info --recursive --size --closure-size --human-readable nixpkgs#rustc
/nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M
/nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M
…
@ -34,7 +34,7 @@ R""(
* Check the existence of a path in a binary cache:
```console
# nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/
# nix path-info --recursive /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/
path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid
```
@ -57,7 +57,7 @@ R""(
size:
```console
# nix path-info --json --all -S \
# nix path-info --json --all --closure-size \
| jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'
[
…,

49
src/nix/profile.cc
View file

@ -31,6 +31,11 @@ struct ProfileElementSource
std::tuple(originalRef.to_string(), attrPath, outputs) <
std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs);
}
std::string to_string() const
{
return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string());
}
};
const int defaultPriority = 5;
@ -42,16 +47,30 @@ struct ProfileElement
bool active = true;
int priority = defaultPriority;
std::string describe() const
std::string identifier() const
{
if (source)
return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string());
return source->to_string();
StringSet names;
for (auto & path : storePaths)
names.insert(DrvName(path.name()).name);
return concatStringsSep(", ", names);
}
/**
* Return a string representing an installable corresponding to the current
* element, either a flakeref or a plain store path
*/
std::set<std::string> toInstallables(Store & store)
{
if (source)
return {source->to_string()};
StringSet rawPaths;
for (auto & path : storePaths)
rawPaths.insert(store.printStorePath(path));
return rawPaths;
}
std::string versions() const
{
StringSet versions;
@ -62,7 +81,7 @@ struct ProfileElement
bool operator < (const ProfileElement & other) const
{
return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths);
}
void updateStorePaths(
@ -237,13 +256,13 @@ struct ProfileManifest
bool changes = false;
while (i != prevElems.end() || j != curElems.end()) {
if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) {
logger->cout("%s%s: ∅ -> %s", indent, j->describe(), j->versions());
if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) {
logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions());
changes = true;
++j;
}
else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) {
logger->cout("%s%s: %s -> ∅", indent, i->describe(), i->versions());
else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) {
logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions());
changes = true;
++i;
}
@ -251,7 +270,7 @@ struct ProfileManifest
auto v1 = i->versions();
auto v2 = j->versions();
if (v1 != v2) {
logger->cout("%s%s: %s -> %s", indent, i->describe(), v1, v2);
logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2);
changes = true;
}
++i;
@ -363,10 +382,10 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
auto profileElement = *it;
for (auto & storePath : profileElement.storePaths) {
if (conflictError.fileA.starts_with(store->printStorePath(storePath))) {
return std::pair(conflictError.fileA, profileElement.source->originalRef);
return std::pair(conflictError.fileA, profileElement.toInstallables(*store));
}
if (conflictError.fileB.starts_with(store->printStorePath(storePath))) {
return std::pair(conflictError.fileB, profileElement.source->originalRef);
return std::pair(conflictError.fileB, profileElement.toInstallables(*store));
}
}
}
@ -375,9 +394,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
// There are 2 conflicting files. We need to find out which one is from the already installed package and
// which one is the package that is the new package that is being installed.
// The first matching package is the one that was already installed (original).
auto [originalConflictingFilePath, originalConflictingRef] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end());
auto [originalConflictingFilePath, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end());
// The last matching package is the one that was going to be installed (new).
auto [newConflictingFilePath, newConflictingRef] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend());
auto [newConflictingFilePath, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend());
throw Error(
"An existing package already provides the following file:\n"
@ -403,8 +422,8 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
" nix profile install %4% --priority %7%\n",
originalConflictingFilePath,
newConflictingFilePath,
originalConflictingRef.to_string(),
newConflictingRef.to_string(),
concatStringsSep(" ", originalConflictingRefs),
concatStringsSep(" ", newConflictingRefs),
conflictError.priority,
conflictError.priority - 1,
conflictError.priority + 1
@ -491,7 +510,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
if (!matches(*store, element, i, matchers)) {
newManifest.elements.push_back(std::move(element));
} else {
notice("removing '%s'", element.describe());
notice("removing '%s'", element.identifier());
}
}

4
src/nix/search.md
View file

@ -52,12 +52,12 @@ R""(
* Search for packages containing `neovim` but hide ones containing either `gui` or `python`:
```console
# nix search nixpkgs neovim -e 'python|gui'
# nix search nixpkgs neovim --exclude 'python|gui'
```
or
```console
# nix search nixpkgs neovim -e 'python' -e 'gui'
# nix search nixpkgs neovim --exclude 'python' --exclude 'gui'
```
# Description

8
src/nix/shell.md
View file

@ -19,26 +19,26 @@ R""(
* Run GNU Hello:
```console
# nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'
# nix shell nixpkgs#hello --command hello --greeting 'Hi everybody!'
Hi everybody!
```
* Run multiple commands in a shell environment:
```console
# nix shell nixpkgs#gnumake -c sh -c "cd src && make"
# nix shell nixpkgs#gnumake --command sh --command "cd src && make"
```
* Run GNU Hello in a chroot store:
```console
# nix shell --store ~/my-nix nixpkgs#hello -c hello
# nix shell --store ~/my-nix nixpkgs#hello --command hello
```
* Start a shell providing GNU Hello in a chroot store:
```console
# nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash
# nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive --command bash
```
Note that it's necessary to specify `bash` explicitly because your

4
src/nix/store-ls.md
View file

@ -5,7 +5,7 @@ R""(
* To list the contents of a store path in a binary cache:
```console
# nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
# nix store ls --store https://cache.nixos.org/ --long --recursive /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
dr-xr-xr-x 0 ./bin
-r-xr-xr-x 38184 ./bin/hello
dr-xr-xr-x 0 ./share
@ -15,7 +15,7 @@ R""(
* To show information about a specific file in a binary cache:
```console
# nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello
# nix store ls --store https://cache.nixos.org/ --long /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello
-r-xr-xr-x 38184 hello
```

2
src/nix/upgrade-nix.md
View file

@ -11,7 +11,7 @@ R""(
* Upgrade Nix in a specific profile:
```console
# nix upgrade-nix -p ~alice/.local/state/nix/profiles/profile
# nix upgrade-nix --profile ~alice/.local/state/nix/profiles/profile
```
# Description

2
src/nix/verify.md
View file

@ -12,7 +12,7 @@ R""(
signatures:
```console
# nix store verify -r -n2 --no-contents $(type -p firefox)
# nix store verify --recursive --sigs-needed 2 --no-contents $(type -p firefox)
```
* Verify a store path in the binary cache `https://cache.nixos.org/`: