1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-17 16:02:43 +01:00

Merge branch 'master' into referenceablePaths

This commit is contained in:
Théophane Hufschmitt 2023-01-30 10:31:00 +01:00 committed by GitHub
commit 4aaf0ee52e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
145 changed files with 4215 additions and 1322 deletions

View file

@ -502,22 +502,9 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
writeNarInfo(narInfo);
}
std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath & path)
{
auto drvPath = path;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (!info->deriver) return std::nullopt;
drvPath = *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath)));
auto logPath = "log/" + std::string(baseNameOf(printStorePath(path)));
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);

View file

@ -129,7 +129,7 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
std::optional<std::string> getBuildLog(const StorePath & path) override;
std::optional<std::string> getBuildLogExact(const StorePath & path) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;

View file

@ -63,7 +63,7 @@
namespace nix {
DerivationGoal::DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(true)
, drvPath(drvPath)
@ -82,7 +82,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(false)
, drvPath(drvPath)
@ -142,18 +142,12 @@ void DerivationGoal::work()
(this->*state)();
}
void DerivationGoal::addWantedOutputs(const StringSet & outputs)
void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
{
/* If we already want all outputs, there is nothing to do. */
if (wantedOutputs.empty()) return;
if (outputs.empty()) {
wantedOutputs.clear();
auto newWanted = wantedOutputs.union_(outputs);
if (!newWanted.isSubsetOf(wantedOutputs))
needRestart = true;
} else
for (auto & i : outputs)
if (wantedOutputs.insert(i).second)
needRestart = true;
wantedOutputs = newWanted;
}
@ -390,7 +384,7 @@ void DerivationGoal::repairClosure()
auto outputs = queryDerivationOutputMap();
StorePathSet outputClosure;
for (auto & i : outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue;
if (!wantedOutputs.contains(i.first)) continue;
worker.store.computeFSClosure(i.second, outputClosure);
}
@ -422,7 +416,7 @@ void DerivationGoal::repairClosure()
if (drvPath2 == outputsToDrv.end())
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair));
}
if (waitees.empty()) {
@ -544,7 +538,8 @@ void DerivationGoal::inputsRealised()
However, the impure derivations feature still relies on this
fragile way of doing things, because its builds do not have
a representation in the store, which is a usability problem
in itself */
in itself. When implementing this logic entirely with lookups
make sure that they're cached. */
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
worker.store.computeFSClosure(*outPath, inputPaths);
}
@ -990,10 +985,15 @@ void DerivationGoal::resolvedFinished()
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames();
// `wantedOutputs` might merely indicate “all the outputs”
auto realWantedOutputs = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return resolvedDrv.outputNames();
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, wantedOutputs.raw());
for (auto & wantedOutput : realWantedOutputs) {
auto initialOutput = get(initialOutputs, wantedOutput);
@ -1321,7 +1321,14 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
if (!drv->type().isPure()) return { false, {} };
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs;
auto wantedOutputsLeft = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return StringSet {};
},
[&](const OutputsSpec::Names & names) {
return static_cast<StringSet>(names);
},
}, wantedOutputs.raw());
DrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) {
@ -1330,7 +1337,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
// this is an invalid output, gets catched with (!wantedOutputsLeft.empty())
continue;
auto & info = *initialOutput;
info.wanted = wantOutput(i.first, wantedOutputs);
info.wanted = wantedOutputs.contains(i.first);
if (info.wanted)
wantedOutputsLeft.erase(i.first);
if (i.second) {
@ -1368,7 +1375,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
}
// If we requested all the outputs via the empty set, we are always fine.
// If we requested all the outputs, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
if (!wantedOutputsLeft.empty())

View file

@ -2,6 +2,7 @@
#include "parsed-derivations.hh"
#include "lock.hh"
#include "outputs-spec.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "goal.hh"
@ -55,7 +56,7 @@ struct DerivationGoal : public Goal
/* The specific outputs that we need to build. Empty means all of
them. */
StringSet wantedOutputs;
OutputsSpec wantedOutputs;
/* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency
@ -128,10 +129,10 @@ struct DerivationGoal : public Goal
std::string machineName;
DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker,
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker,
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
virtual ~DerivationGoal();
@ -142,7 +143,7 @@ struct DerivationGoal : public Goal
void work() override;
/* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs);
void addWantedOutputs(const OutputsSpec & outputs);
/* The states. */
void getDerivation();

View file

@ -80,7 +80,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
BuildMode buildMode)
{
Worker worker(*this, *this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
try {
worker.run(Goals{goal});
@ -89,7 +89,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
return BuildResult {
.status = BuildResult::MiscFailure,
.errorMsg = e.msg(),
.path = DerivedPath::Built { .drvPath = drvPath },
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
};
}
@ -130,7 +133,8 @@ void LocalStore::repairPath(const StorePath & path)
auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) {
goals.clear();
goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
// FIXME: Should just build the specific output we need.
goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair));
worker.run(goals);
} else
throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));

View file

@ -1459,7 +1459,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
unknown, downloadSize, narSize);
}
virtual std::optional<std::string> getBuildLog(const StorePath & path) override
virtual std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ return std::nullopt; }
virtual void addBuildLog(const StorePath & path, std::string_view log) override
@ -2050,10 +2050,14 @@ void LocalDerivationGoal::runChild()
sandboxProfile += "(deny default (with no-log))\n";
}
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
sandboxProfile +=
#include "sandbox-defaults.sb"
;
if (!derivationType.isSandboxed())
sandboxProfile += "(import \"sandbox-network.sb\")\n";
sandboxProfile +=
#include "sandbox-network.sb"
;
/* Add the output paths we'll use at build-time to the chroot */
sandboxProfile += "(allow file-read* file-write* process-exec\n";
@ -2096,7 +2100,9 @@ void LocalDerivationGoal::runChild()
sandboxProfile += additionalSandboxProfile;
} else
sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
sandboxProfile +=
#include "sandbox-minimal.sb"
;
debug("Generated sandbox profile:");
debug(sandboxProfile);
@ -2121,8 +2127,6 @@ void LocalDerivationGoal::runChild()
args.push_back(sandboxFile);
args.push_back("-D");
args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
args.push_back("-D");
args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
if (allowLocalNetworking) {
args.push_back("-D");
args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));
@ -2748,7 +2752,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
if (wantOutput(outputName, wantedOutputs))
if (wantedOutputs.contains(outputName))
builtOutputs.emplace(thisRealisation.id, thisRealisation);
}

View file

@ -1,3 +1,5 @@
R""(
(define TMPDIR (param "_GLOBAL_TMP_DIR"))
(deny default)
@ -104,3 +106,5 @@
(subpath "/System/Library/Apple/usr/libexec/oah")
(subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist")
(subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist"))
)""

View file

@ -1,5 +1,9 @@
R""(
(allow default)
; Disallow creating setuid/setgid binaries, since that
; would allow breaking build user isolation.
(deny file-write-setugid)
)""

View file

@ -1,3 +1,5 @@
R""(
; Allow local and remote network traffic.
(allow network* (local ip) (remote ip))
@ -18,3 +20,5 @@
; Allow access to trustd.
(allow mach-lookup (global-name "com.apple.trustd"))
(allow mach-lookup (global-name "com.apple.trustd.agent"))
)""

View file

@ -42,7 +42,7 @@ Worker::~Worker()
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
const StringSet & wantedOutputs,
const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
{
std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
@ -59,7 +59,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode)
const OutputsSpec & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
@ -70,7 +70,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)

View file

@ -140,15 +140,15 @@ public:
/* derivation goal */
private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const StringSet & wantedOutputs,
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
public:
std::shared_ptr<DerivationGoal> makeDerivationGoal(
const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);

View file

@ -222,7 +222,8 @@ struct ClientSettings
else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
subs.push_back(s + "/");
else
warn("ignoring untrusted substituter '%s'", s);
warn("ignoring untrusted substituter '%s', you are not a trusted user.\n"
"Run `man nix.conf` for more information on the `substituters` configuration option.", s);
res = subs;
return true;
};

View file

@ -688,12 +688,6 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
}
bool wantOutput(const std::string & output, const std::set<std::string> & wanted)
{
return wanted.empty() || wanted.find(output) != wanted.end();
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
const auto pathS = readString(in);

View file

@ -13,6 +13,7 @@
namespace nix {
class Store;
/* Abstract syntax of derivations. */
@ -294,8 +295,6 @@ typedef std::map<StorePath, DrvHash> DrvHashes;
// FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes;
bool wantOutput(const std::string & output, const std::set<std::string> & wanted);
struct Source;
struct Sink;

View file

@ -19,11 +19,11 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
res["drvPath"] = store->printStorePath(drvPath);
// Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so lets do it
const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
for (const auto & output : outputs) {
auto knownOutput = get(knownOutputs, output);
if (knownOutput && *knownOutput)
res["outputs"][output] = store->printStorePath(**knownOutput);
const auto outputMap = store->queryPartialDerivationOutputMap(drvPath);
for (const auto & [output, outputPathOpt] : outputMap) {
if (!outputs.contains(output)) continue;
if (outputPathOpt)
res["outputs"][output] = store->printStorePath(*outputPathOpt);
else
res["outputs"][output] = nullptr;
}
@ -63,7 +63,7 @@ std::string DerivedPath::Built::to_string(const Store & store) const
{
return store.printStorePath(drvPath)
+ "!"
+ (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
+ outputs.to_string();
}
std::string DerivedPath::to_string(const Store & store) const
@ -81,15 +81,10 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_
DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS)
{
auto drvPath = store.parseStorePath(drvS);
std::set<std::string> outputs;
if (outputsS != "*") {
outputs = tokenizeString<std::set<std::string>>(outputsS, ",");
if (outputs.empty())
throw Error(
"Explicit list of wanted outputs '%s' must not be empty. Consider using '*' as a wildcard meaning all outputs if no output in particular is wanted.", outputsS);
}
return {drvPath, outputs};
return {
.drvPath = store.parseStorePath(drvS),
.outputs = OutputsSpec::parse(outputsS),
};
}
DerivedPath DerivedPath::parse(const Store & store, std::string_view s)

View file

@ -3,6 +3,7 @@
#include "util.hh"
#include "path.hh"
#include "realisation.hh"
#include "outputs-spec.hh"
#include <optional>
@ -44,7 +45,7 @@ struct DerivedPathOpaque {
*/
struct DerivedPathBuilt {
StorePath drvPath;
std::set<std::string> outputs;
OutputsSpec outputs;
std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);

View file

@ -77,60 +77,73 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
}
void LocalStore::addTempRoot(const StorePath & path)
void LocalStore::createTempRootsFile()
{
auto state(_state.lock());
auto fdTempRoots(_fdTempRoots.lock());
/* Create the temporary roots file for this process. */
if (!state->fdTempRoots) {
if (*fdTempRoots) return;
while (1) {
if (pathExists(fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
unlink(fnTempRoots.c_str());
while (1) {
if (pathExists(fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
unlink(fnTempRoots.c_str());
state->fdTempRoots = openLockFile(fnTempRoots, true);
*fdTempRoots = openLockFile(fnTempRoots, true);
debug("acquiring write lock on '%s'", fnTempRoots);
lockFile(state->fdTempRoots.get(), ltWrite, true);
debug("acquiring write lock on '%s'", fnTempRoots);
lockFile(fdTempRoots->get(), ltWrite, true);
/* Check whether the garbage collector didn't get in our
way. */
struct stat st;
if (fstat(state->fdTempRoots.get(), &st) == -1)
throw SysError("statting '%1%'", fnTempRoots);
if (st.st_size == 0) break;
/* Check whether the garbage collector didn't get in our
way. */
struct stat st;
if (fstat(fdTempRoots->get(), &st) == -1)
throw SysError("statting '%1%'", fnTempRoots);
if (st.st_size == 0) break;
/* The garbage collector deleted this file before we could
get a lock. (It won't delete the file after we get a
lock.) Try again. */
}
/* The garbage collector deleted this file before we could get
a lock. (It won't delete the file after we get a lock.)
Try again. */
}
}
void LocalStore::addTempRoot(const StorePath & path)
{
createTempRootsFile();
/* Open/create the global GC lock file. */
{
auto fdGCLock(_fdGCLock.lock());
if (!*fdGCLock)
*fdGCLock = openGCLock();
}
if (!state->fdGCLock)
state->fdGCLock = openGCLock();
restart:
FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
/* Try to acquire a shared global GC lock (non-blocking). This
only succeeds if the garbage collector is not currently
running. */
FdLock gcLock(_fdGCLock.lock()->get(), ltRead, false, "");
if (!gcLock.acquired) {
/* We couldn't get a shared global GC lock, so the garbage
collector is running. So we have to connect to the garbage
collector and inform it about our root. */
if (!state->fdRootsSocket) {
auto fdRootsSocket(_fdRootsSocket.lock());
if (!*fdRootsSocket) {
auto socketPath = stateDir.get() + gcSocketPath;
debug("connecting to '%s'", socketPath);
state->fdRootsSocket = createUnixDomainSocket();
*fdRootsSocket = createUnixDomainSocket();
try {
nix::connect(state->fdRootsSocket.get(), socketPath);
nix::connect(fdRootsSocket->get(), socketPath);
} catch (SysError & e) {
/* The garbage collector may have exited, so we need to
restart. */
if (e.errNo == ECONNREFUSED) {
debug("GC socket connection refused");
state->fdRootsSocket.close();
fdRootsSocket->close();
goto restart;
}
throw;
@ -139,9 +152,9 @@ void LocalStore::addTempRoot(const StorePath & path)
try {
debug("sending GC root '%s'", printStorePath(path));
writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false);
char c;
readFull(state->fdRootsSocket.get(), &c, 1);
readFull(fdRootsSocket->get(), &c, 1);
assert(c == '1');
debug("got ack for GC root '%s'", printStorePath(path));
} catch (SysError & e) {
@ -149,20 +162,21 @@ void LocalStore::addTempRoot(const StorePath & path)
restart. */
if (e.errNo == EPIPE || e.errNo == ECONNRESET) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
fdRootsSocket->close();
goto restart;
}
throw;
} catch (EndOfFile & e) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
fdRootsSocket->close();
goto restart;
}
}
/* Append the store path to the temporary roots file. */
/* Record the store path in the temporary roots file so it will be
seen by a future run of the garbage collector. */
auto s = printStorePath(path) + '\0';
writeFull(state->fdTempRoots.get(), s);
writeFull(_fdTempRoots.lock()->get(), s);
}

View file

@ -570,11 +570,15 @@ public:
{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="},
"trusted-public-keys",
R"(
A whitespace-separated list of public keys. When paths are copied
from another Nix store (such as a binary cache), they must be
signed with one of these keys. For example:
`cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`.
A whitespace-separated list of public keys.
At least one of the following condition must be met
for Nix to accept copying a store object from another
Nix store (such as a substituter):
- the store object has been signed using a key in the trusted keys list
- the [`require-sigs`](#conf-require-sigs) option has been set to `false`
- the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object)
)",
{"binary-cache-public-keys"}};
@ -670,13 +674,14 @@ public:
independently. Lower value means higher priority.
The default is `https://cache.nixos.org`, with a Priority of 40.
Nix will copy a store path from a remote store only if one
of the following is true:
At least one of the following conditions must be met for Nix to use
a substituter:
- the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys)
- the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
- the [`require-sigs`](#conf-require-sigs) option has been set to `false`
- the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object)
- the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
In addition, each store path should be trusted as described
in [`trusted-public-keys`](#conf-trusted-public-keys)
)",
{"binary-caches"}};

View file

@ -279,7 +279,12 @@ public:
conn->to.flush();
BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } };
BuildResult status {
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg;

View file

@ -87,20 +87,8 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
const std::string LocalFSStore::drvsLogDir = "drvs";
std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path)
{
auto path = path_;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return std::nullopt;
path = *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
auto baseName = path.to_string();
for (int j = 0; j < 2; j++) {

View file

@ -50,7 +50,7 @@ public:
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
}
std::optional<std::string> getBuildLog(const StorePath & path) override;
std::optional<std::string> getBuildLogExact(const StorePath & path) override;
};

View file

@ -441,9 +441,9 @@ LocalStore::~LocalStore()
}
try {
auto state(_state.lock());
if (state->fdTempRoots) {
state->fdTempRoots = -1;
auto fdTempRoots(_fdTempRoots.lock());
if (*fdTempRoots) {
*fdTempRoots = -1;
unlink(fnTempRoots.c_str());
}
} catch (...) {

View file

@ -59,15 +59,6 @@ private:
struct Stmts;
std::unique_ptr<Stmts> stmts;
/* The global GC lock */
AutoCloseFD fdGCLock;
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
/* Connection to the garbage collector. */
AutoCloseFD fdRootsSocket;
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@ -156,6 +147,21 @@ public:
void addTempRoot(const StorePath & path) override;
private:
void createTempRootsFile();
/* The file to which we write our temporary roots. */
Sync<AutoCloseFD> _fdTempRoots;
/* The global GC lock. */
Sync<AutoCloseFD> _fdGCLock;
/* Connection to the garbage collector. */
Sync<AutoCloseFD> _fdRootsSocket;
public:
void addIndirectRoot(const Path & path) override;
private:

View file

@ -13,10 +13,6 @@ ifdef HOST_LINUX
libstore_LDFLAGS += -ldl
endif
ifdef HOST_DARWIN
libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
endif
$(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
ifeq ($(ENABLE_S3), 1)

View file

@ -123,8 +123,12 @@ struct AutoUserLock : UserLock
std::vector<gid_t> getSupplementaryGIDs() override { return {}; }
static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useChroot)
static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useUserNamespace)
{
#if !defined(__linux__)
useUserNamespace = false;
#endif
settings.requireExperimentalFeature(Xp::AutoAllocateUids);
assert(settings.startId > 0);
assert(settings.uidCount % maxIdsPerBuild == 0);
@ -157,7 +161,7 @@ struct AutoUserLock : UserLock
auto lock = std::make_unique<AutoUserLock>();
lock->fdUserLock = std::move(fd);
lock->firstUid = firstUid;
if (useChroot)
if (useUserNamespace)
lock->firstGid = firstUid;
else {
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@ -174,10 +178,10 @@ struct AutoUserLock : UserLock
}
};
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot)
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace)
{
if (settings.autoAllocateUids)
return AutoUserLock::acquire(nrIds, useChroot);
return AutoUserLock::acquire(nrIds, useUserNamespace);
else
return SimpleUserLock::acquire();
}

View file

@ -31,7 +31,7 @@ struct UserLock
/* Acquire a user lock for a UID range of size `nrIds`. Note that this
may return nullptr if no user is available. */
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot);
std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace);
bool useBuildUsers();

12
src/libstore/log-store.cc Normal file
View file

@ -0,0 +1,12 @@
#include "log-store.hh"
namespace nix {
std::optional<std::string> LogStore::getBuildLog(const StorePath & path) {
auto maybePath = getBuildDerivationPath(path);
if (!maybePath)
return std::nullopt;
return getBuildLogExact(maybePath.value());
}
}

View file

@ -11,7 +11,9 @@ struct LogStore : public virtual Store
/* Return the build log of the specified store path, if available,
or null otherwise. */
virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0;
std::optional<std::string> getBuildLog(const StorePath & path);
virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0;
virtual void addBuildLog(const StorePath & path, std::string_view log) = 0;

View file

@ -185,7 +185,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
knownOutputPaths = false;
break;
}
if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt))
if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt))
invalid.insert(*pathOpt);
}
if (knownOutputPaths && invalid.empty()) return;
@ -301,4 +301,47 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : store;
OutputPathMap outputs;
auto drv = evalStore.readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(store, drv);
auto drvOutputs = drv.outputsAndOptPaths(store);
auto outputNames = std::visit(overloaded {
[&](const OutputsSpec::All &) {
StringSet names;
for (auto & [outputName, _] : drv.outputs)
names.insert(outputName);
return names;
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, bfd.outputs.raw());
for (auto & output : outputNames) {
auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store.printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
DrvOutput outputId { *outputHash, output };
auto realisation = store.queryRealisation(outputId);
if (!realisation)
throw MissingRealisation(outputId);
outputs.insert_or_assign(output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
outputs.insert_or_assign(output, *drvOutput->second);
}
}
return outputs;
}
}

View file

@ -0,0 +1,195 @@
#include <regex>
#include <nlohmann/json.hpp>
#include "util.hh"
#include "regex-combinators.hh"
#include "outputs-spec.hh"
#include "path-regex.hh"
namespace nix {
bool OutputsSpec::contains(const std::string & outputName) const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) {
return true;
},
[&](const OutputsSpec::Names & outputNames) {
return outputNames.count(outputName) > 0;
},
}, raw());
}
static std::string outputSpecRegexStr =
regex::either(
regex::group(R"(\*)"),
regex::group(regex::list(nameRegexStr)));
std::optional<OutputsSpec> OutputsSpec::parseOpt(std::string_view s)
{
static std::regex regex(std::string { outputSpecRegexStr });
std::smatch match;
std::string s2 { s }; // until some improves std::regex
if (!std::regex_match(s2, match, regex))
return std::nullopt;
if (match[1].matched)
return { OutputsSpec::All {} };
if (match[2].matched)
return OutputsSpec::Names { tokenizeString<StringSet>(match[2].str(), ",") };
assert(false);
}
OutputsSpec OutputsSpec::parse(std::string_view s)
{
std::optional spec = parseOpt(s);
if (!spec)
throw Error("invalid outputs specifier '%s'", s);
return *spec;
}
std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> ExtendedOutputsSpec::parseOpt(std::string_view s)
{
auto found = s.rfind('^');
if (found == std::string::npos)
return std::pair { s, ExtendedOutputsSpec::Default {} };
auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1));
if (!specOpt)
return std::nullopt;
return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } };
}
std::pair<std::string_view, ExtendedOutputsSpec> ExtendedOutputsSpec::parse(std::string_view s)
{
std::optional spec = parseOpt(s);
if (!spec)
throw Error("invalid extended outputs specifier '%s'", s);
return *spec;
}
std::string OutputsSpec::to_string() const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) -> std::string {
return "*";
},
[&](const OutputsSpec::Names & outputNames) -> std::string {
return concatStringsSep(",", outputNames);
},
}, raw());
}
std::string ExtendedOutputsSpec::to_string() const
{
return std::visit(overloaded {
[&](const ExtendedOutputsSpec::Default &) -> std::string {
return "";
},
[&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string {
return "^" + outputSpec.to_string();
},
}, raw());
}
OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) -> OutputsSpec {
return OutputsSpec::All { };
},
[&](const OutputsSpec::Names & theseNames) -> OutputsSpec {
return std::visit(overloaded {
[&](const OutputsSpec::All &) -> OutputsSpec {
return OutputsSpec::All {};
},
[&](const OutputsSpec::Names & thoseNames) -> OutputsSpec {
OutputsSpec::Names ret = theseNames;
ret.insert(thoseNames.begin(), thoseNames.end());
return ret;
},
}, that.raw());
},
}, raw());
}
bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const
{
return std::visit(overloaded {
[&](const OutputsSpec::All &) {
return true;
},
[&](const OutputsSpec::Names & thoseNames) {
return std::visit(overloaded {
[&](const OutputsSpec::All &) {
return false;
},
[&](const OutputsSpec::Names & theseNames) {
bool ret = true;
for (auto & o : theseNames)
if (thoseNames.count(o) == 0)
ret = false;
return ret;
},
}, raw());
},
}, that.raw());
}
}
namespace nlohmann {
using namespace nix;
OutputsSpec adl_serializer<OutputsSpec>::from_json(const json & json) {
auto names = json.get<StringSet>();
if (names == StringSet({"*"}))
return OutputsSpec::All {};
else
return OutputsSpec::Names { std::move(names) };
}
void adl_serializer<OutputsSpec>::to_json(json & json, OutputsSpec t) {
std::visit(overloaded {
[&](const OutputsSpec::All &) {
json = std::vector<std::string>({"*"});
},
[&](const OutputsSpec::Names & names) {
json = names;
},
}, t.raw());
}
ExtendedOutputsSpec adl_serializer<ExtendedOutputsSpec>::from_json(const json & json) {
if (json.is_null())
return ExtendedOutputsSpec::Default {};
else {
return ExtendedOutputsSpec::Explicit { json.get<OutputsSpec>() };
}
}
void adl_serializer<ExtendedOutputsSpec>::to_json(json & json, ExtendedOutputsSpec t) {
std::visit(overloaded {
[&](const ExtendedOutputsSpec::Default &) {
json = nullptr;
},
[&](const ExtendedOutputsSpec::Explicit & e) {
adl_serializer<OutputsSpec>::to_json(json, e);
},
}, t.raw());
}
}

View file

@ -0,0 +1,95 @@
#pragma once
#include <cassert>
#include <optional>
#include <set>
#include <variant>
#include "json-impls.hh"
namespace nix {
struct OutputNames : std::set<std::string> {
using std::set<std::string>::set;
/* These need to be "inherited manually" */
OutputNames(const std::set<std::string> & s)
: std::set<std::string>(s)
{ assert(!empty()); }
OutputNames(std::set<std::string> && s)
: std::set<std::string>(s)
{ assert(!empty()); }
/* This set should always be non-empty, so we delete this
constructor in order make creating empty ones by mistake harder.
*/
OutputNames() = delete;
};
struct AllOutputs : std::monostate { };
typedef std::variant<AllOutputs, OutputNames> _OutputsSpecRaw;
struct OutputsSpec : _OutputsSpecRaw {
using Raw = _OutputsSpecRaw;
using Raw::Raw;
/* Force choosing a variant */
OutputsSpec() = delete;
using Names = OutputNames;
using All = AllOutputs;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
inline Raw & raw() {
return static_cast<Raw &>(*this);
}
bool contains(const std::string & output) const;
/* Create a new OutputsSpec which is the union of this and that. */
OutputsSpec union_(const OutputsSpec & that) const;
/* Whether this OutputsSpec is a subset of that. */
bool isSubsetOf(const OutputsSpec & outputs) const;
/* Parse a string of the form 'output1,...outputN' or
'*', returning the outputs spec. */
static OutputsSpec parse(std::string_view s);
static std::optional<OutputsSpec> parseOpt(std::string_view s);
std::string to_string() const;
};
struct DefaultOutputs : std::monostate { };
typedef std::variant<DefaultOutputs, OutputsSpec> _ExtendedOutputsSpecRaw;
struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw {
using Raw = _ExtendedOutputsSpecRaw;
using Raw::Raw;
using Default = DefaultOutputs;
using Explicit = OutputsSpec;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the extended outputs spec. */
static std::pair<std::string_view, ExtendedOutputsSpec> parse(std::string_view s);
static std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> parseOpt(std::string_view s);
std::string to_string() const;
};
}
JSON_IMPL(OutputsSpec)
JSON_IMPL(ExtendedOutputsSpec)

View file

@ -3,6 +3,80 @@
namespace nix {
std::string ValidPathInfo::fingerprint(const Store & store) const
{
if (narSize == 0)
throw Error("cannot calculate fingerprint of path '%s' because its size is not known",
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references));
}
void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
{
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;
auto caPath = std::visit(overloaded {
[&](const TextHash & th) {
return store.makeTextPath(path.name(), th.hash, references);
},
[&](const FixedOutputHash & fsh) {
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
hasSelfReference = true;
refs.erase(path);
}
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference);
}
}, *ca);
bool res = caPath == path;
if (!res)
printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
return res;
}
size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
{
if (isContentAddressed(store)) return maxSigs;
size_t good = 0;
for (auto & sig : sigs)
if (checkSignature(store, publicKeys, sig))
good++;
return good;
}
bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const
{
return verifyDetached(fingerprint(store), sig, publicKeys);
}
Strings ValidPathInfo::shortRefs() const
{
Strings refs;
for (auto & r : references)
refs.push_back(std::string(r.to_string()));
return refs;
}
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
{
return read(source, store, format, store.parseStorePath(readString(source)));
@ -24,6 +98,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
return info;
}
void ValidPathInfo::write(
Sink & sink,
const Store & store,

View file

@ -0,0 +1,7 @@
#pragma once
namespace nix {
static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)";
}

View file

@ -1,6 +1,5 @@
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "nlohmann/json.hpp"
#include <regex>
@ -16,10 +15,14 @@ std::string StorePathWithOutputs::to_string(const Store & store) const
DerivedPath StorePathWithOutputs::toDerivedPath() const
{
if (!outputs.empty() || path.isDerivation())
return DerivedPath::Built { path, outputs };
else
if (!outputs.empty()) {
return DerivedPath::Built { path, OutputsSpec::Names { outputs } };
} else if (path.isDerivation()) {
assert(outputs.empty());
return DerivedPath::Built { path, OutputsSpec::All { } };
} else {
return DerivedPath::Opaque { path };
}
}
@ -42,7 +45,18 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDeriv
return StorePathWithOutputs { bo.path };
},
[&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> {
return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
return StorePathWithOutputs {
.path = bfd.drvPath,
// Use legacy encoding of wildcard as empty set
.outputs = std::visit(overloaded {
[&](const OutputsSpec::All &) -> StringSet {
return {};
},
[&](const OutputsSpec::Names & outputs) {
return static_cast<StringSet>(outputs);
},
}, bfd.outputs.raw()),
};
},
}, p.raw());
}
@ -53,8 +67,8 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<std::string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<std::string>>(((std::string_view) s).substr(n + 1), ","));
: std::make_pair(s.substr(0, n),
tokenizeString<std::set<std::string>>(s.substr(n + 1), ","));
}
@ -71,57 +85,4 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
}
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
{
static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
std::smatch match;
if (!std::regex_match(s, match, regex))
return {s, DefaultOutputs()};
if (match[3].matched)
return {match[1], AllOutputs()};
return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
}
std::string printOutputsSpec(const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
return "";
if (std::get_if<AllOutputs>(&outputsSpec))
return "^*";
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
return "^" + concatStringsSep(",", *outputNames);
assert(false);
}
void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec)
{
if (std::get_if<DefaultOutputs>(&outputsSpec))
json = nullptr;
else if (std::get_if<AllOutputs>(&outputsSpec))
json = std::vector<std::string>({"*"});
else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
json = *outputNames;
}
void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec)
{
if (json.is_null())
outputsSpec = DefaultOutputs();
else {
auto names = json.get<OutputNames>();
if (names == OutputNames({"*"}))
outputsSpec = AllOutputs();
else
outputsSpec = names;
}
}
}

View file

@ -1,13 +1,17 @@
#pragma once
#include <variant>
#include "path.hh"
#include "derived-path.hh"
#include "nlohmann/json_fwd.hpp"
namespace nix {
/* This is a deprecated old type just for use by the old CLI, and older
versions of the RPC protocols. In new code don't use it; you want
`DerivedPath` instead.
`DerivedPath` is better because it handles more cases, and does so more
explicitly without devious punning tricks.
*/
struct StorePathWithOutputs
{
StorePath path;
@ -33,25 +37,4 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
typedef std::set<std::string> OutputNames;
struct AllOutputs {
bool operator < (const AllOutputs & _) const { return false; }
};
struct DefaultOutputs {
bool operator < (const DefaultOutputs & _) const { return false; }
};
typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the outputs spec. */
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
std::string printOutputsSpec(const OutputsSpec & outputsSpec);
void to_json(nlohmann::json &, const OutputsSpec &);
void from_json(const nlohmann::json &, OutputsSpec &);
}

View file

@ -8,8 +8,10 @@ static void checkName(std::string_view path, std::string_view name)
{
if (name.empty())
throw BadStorePath("store path '%s' has an empty name", path);
if (name.size() > 211)
throw BadStorePath("store path '%s' has a name longer than 211 characters", path);
if (name.size() > StorePath::MaxPathLen)
throw BadStorePath("store path '%s' has a name longer than '%d characters",
StorePath::MaxPathLen, path);
// See nameRegexStr for the definition
for (auto c : name)
if (!((c >= '0' && c <= '9')
|| (c >= 'a' && c <= 'z')

View file

@ -5,7 +5,6 @@
namespace nix {
class Store;
struct Hash;
class StorePath
@ -17,6 +16,8 @@ public:
/* Size of the hash part of store paths, in base-32 characters. */
constexpr static size_t HashLen = 32; // i.e. 160 bits
constexpr static size_t MaxPathLen = 211;
StorePath() = delete;
StorePath(std::string_view baseName);
@ -64,7 +65,6 @@ public:
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
typedef std::map<std::string, StorePath> OutputPathMap;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;

View file

@ -7,6 +7,8 @@
namespace nix {
class Store;
struct DrvOutput {
// The hash modulo of the derivation
Hash drvHash;
@ -93,4 +95,14 @@ struct RealisedPath {
GENERATE_CMP(RealisedPath, me->raw);
};
class MissingRealisation : public Error
{
public:
MissingRealisation(DrvOutput & outputId)
: Error( "cannot operate on an output of the "
"unbuilt derivation '%s'",
outputId.to_string())
{}
};
}

View file

@ -867,8 +867,8 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
const auto drvOutputs = drv.outputsAndOptPaths(*this);
for (auto & output : bfd.outputs) {
auto built = resolveDerivedPath(*this, bfd, &*evalStore);
for (auto & [output, outputPath] : built) {
auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error(
@ -879,22 +879,14 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
auto realisation =
queryRealisation(outputId);
if (!realisation)
throw Error(
"cannot operate on an output of unbuilt "
"content-addressed derivation '%s'",
outputId.to_string());
throw MissingRealisation(outputId);
res.builtOutputs.emplace(realisation->id, *realisation);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
const auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
res.builtOutputs.emplace(
outputId,
Realisation {
.id = outputId,
.outPath = *drvOutput->second,
.outPath = outputPath,
});
}
}
@ -918,7 +910,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
writeDerivation(conn->to, *this, drv);
conn->to << buildMode;
conn.processStderr();
BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } };
BuildResult res {
.path = DerivedPath::Built {
.drvPath = drvPath,
.outputs = OutputsSpec::All { },
},
};
res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {

View file

@ -53,8 +53,8 @@ public:
{ return false; }
// FIXME extend daemon protocol, move implementation to RemoteStore
std::optional<std::string> getBuildLog(const StorePath & path) override
{ unsupported("getBuildLog"); }
std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ unsupported("getBuildLogExact"); }
private:

View file

@ -1210,79 +1210,6 @@ std::string showPaths(const PathSet & paths)
}
std::string ValidPathInfo::fingerprint(const Store & store) const
{
if (narSize == 0)
throw Error("cannot calculate fingerprint of path '%s' because its size is not known",
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references));
}
void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
{
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;
auto caPath = std::visit(overloaded {
[&](const TextHash & th) {
return store.makeTextPath(path.name(), th.hash, references);
},
[&](const FixedOutputHash & fsh) {
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
hasSelfReference = true;
refs.erase(path);
}
return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference);
}
}, *ca);
bool res = caPath == path;
if (!res)
printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
return res;
}
size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
{
if (isContentAddressed(store)) return maxSigs;
size_t good = 0;
for (auto & sig : sigs)
if (checkSignature(store, publicKeys, sig))
good++;
return good;
}
bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const
{
return verifyDetached(fingerprint(store), sig, publicKeys);
}
Strings ValidPathInfo::shortRefs() const
{
Strings refs;
for (auto & r : references)
refs.push_back(std::string(r.to_string()));
return refs;
}
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
ensurePath(drvPath);
@ -1301,6 +1228,34 @@ Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool req
}
}
std::optional<StorePath> Store::getBuildDerivationPath(const StorePath & path)
{
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return std::nullopt;
return *info->deriver;
} catch (InvalidPath &) {
return std::nullopt;
}
}
if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path))
return path;
auto drv = readDerivation(path);
if (!drv.type().hasKnownOutputPaths()) {
// The build log is actually attached to the corresponding
// resolved derivation, so we need to get it first
auto resolvedDrv = drv.tryResolve(*this);
if (resolvedDrv)
return writeDerivation(*this, *resolvedDrv, NoRepair, true);
}
return path;
}
Derivation Store::readDerivation(const StorePath & drvPath)
{ return readDerivationCommon(*this, drvPath, true); }

View file

@ -71,6 +71,9 @@ class NarInfoDiskCache;
class Store;
typedef std::map<std::string, StorePath> OutputPathMap;
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
@ -120,6 +123,8 @@ public:
typedef std::map<std::string, std::string> Params;
protected:
struct PathInfoCacheValue {
@ -618,6 +623,13 @@ public:
*/
StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);
/**
* Given a store path, return the realisation actually used in the realisation of this path:
* - If the path is a content-addressed derivation, try to resolve it
* - Otherwise, find one of its derivers
*/
std::optional<StorePath> getBuildDerivationPath(const StorePath &);
/* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */
void clearPathInfoCache()
@ -719,6 +731,11 @@ void copyClosure(
void removeTempRoots();
/* Resolve the derived path completely, failing if any derivation output
is unknown. */
OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr);
/* Return a Store object to access the Nix store denoted by
uri (slight misnomer...). Supported values are:

View file

@ -0,0 +1,23 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "store-api.hh"
namespace nix {
class LibStoreTest : public ::testing::Test {
public:
static void SetUpTestSuite() {
initLibStore();
}
protected:
LibStoreTest()
: store(openStore("dummy://"))
{ }
ref<Store> store;
};
} /* namespace nix */

View file

@ -12,4 +12,4 @@ libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
libstore-tests_LIBS = libstore libutil
libstore-tests_LDFLAGS := $(GTEST_LIBS)
libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

View file

@ -0,0 +1,201 @@
#include "outputs-spec.hh"
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
namespace nix {
#ifndef NDEBUG
TEST(OutputsSpec, no_empty_names) {
ASSERT_DEATH(OutputsSpec::Names { std::set<std::string> { } }, "");
}
#endif
#define TEST_DONT_PARSE(NAME, STR) \
TEST(OutputsSpec, bad_ ## NAME) { \
std::optional OutputsSpecOpt = \
OutputsSpec::parseOpt(STR); \
ASSERT_FALSE(OutputsSpecOpt); \
}
TEST_DONT_PARSE(empty, "")
TEST_DONT_PARSE(garbage, "&*()")
TEST_DONT_PARSE(double_star, "**")
TEST_DONT_PARSE(star_first, "*,foo")
TEST_DONT_PARSE(star_second, "foo,*")
#undef TEST_DONT_PARSE
TEST(OutputsSpec, all) {
std::string_view str = "*";
OutputsSpec expected = OutputsSpec::All { };
ASSERT_EQ(OutputsSpec::parse(str), expected);
ASSERT_EQ(expected.to_string(), str);
}
TEST(OutputsSpec, names_out) {
std::string_view str = "out";
OutputsSpec expected = OutputsSpec::Names { "out" };
ASSERT_EQ(OutputsSpec::parse(str), expected);
ASSERT_EQ(expected.to_string(), str);
}
TEST(OutputsSpec, names_underscore) {
std::string_view str = "a_b";
OutputsSpec expected = OutputsSpec::Names { "a_b" };
ASSERT_EQ(OutputsSpec::parse(str), expected);
ASSERT_EQ(expected.to_string(), str);
}
TEST(OutputsSpec, names_numberic) {
std::string_view str = "01";
OutputsSpec expected = OutputsSpec::Names { "01" };
ASSERT_EQ(OutputsSpec::parse(str), expected);
ASSERT_EQ(expected.to_string(), str);
}
TEST(OutputsSpec, names_out_bin) {
OutputsSpec expected = OutputsSpec::Names { "out", "bin" };
ASSERT_EQ(OutputsSpec::parse("out,bin"), expected);
// N.B. This normalization is OK.
ASSERT_EQ(expected.to_string(), "bin,out");
}
#define TEST_SUBSET(X, THIS, THAT) \
X((OutputsSpec { THIS }).isSubsetOf(THAT));
TEST(OutputsSpec, subsets_all_all) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All { }, OutputsSpec::All { });
}
TEST(OutputsSpec, subsets_names_all) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::All { });
}
TEST(OutputsSpec, subsets_names_names_eq) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::Names { "a" });
}
TEST(OutputsSpec, subsets_names_names_noneq) {
TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, (OutputsSpec::Names { "a", "b" }));
}
TEST(OutputsSpec, not_subsets_all_names) {
TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All { }, OutputsSpec::Names { "a" });
}
TEST(OutputsSpec, not_subsets_names_names) {
TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names { "a", "b" }), (OutputsSpec::Names { "a" }));
}
#undef TEST_SUBSET
#define TEST_UNION(RES, THIS, THAT) \
ASSERT_EQ(OutputsSpec { RES }, (OutputsSpec { THIS }).union_(THAT));
TEST(OutputsSpec, union_all_all) {
TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::All { });
}
TEST(OutputsSpec, union_all_names) {
TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::Names { "a" });
}
TEST(OutputsSpec, union_names_all) {
TEST_UNION(OutputsSpec::All { }, OutputsSpec::Names { "a" }, OutputsSpec::All { });
}
TEST(OutputsSpec, union_names_names) {
TEST_UNION((OutputsSpec::Names { "a", "b" }), OutputsSpec::Names { "a" }, OutputsSpec::Names { "b" });
}
#undef TEST_UNION
#define TEST_DONT_PARSE(NAME, STR) \
TEST(ExtendedOutputsSpec, bad_ ## NAME) { \
std::optional extendedOutputsSpecOpt = \
ExtendedOutputsSpec::parseOpt(STR); \
ASSERT_FALSE(extendedOutputsSpecOpt); \
}
TEST_DONT_PARSE(carot_empty, "^")
TEST_DONT_PARSE(prefix_carot_empty, "foo^")
TEST_DONT_PARSE(garbage, "^&*()")
TEST_DONT_PARSE(double_star, "^**")
TEST_DONT_PARSE(star_first, "^*,foo")
TEST_DONT_PARSE(star_second, "^foo,*")
#undef TEST_DONT_PARSE
TEST(ExtendedOutputsSpec, defeault) {
std::string_view str = "foo";
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = ExtendedOutputsSpec::Default { };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
}
TEST(ExtendedOutputsSpec, all) {
std::string_view str = "foo^*";
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = OutputsSpec::All { };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
}
TEST(ExtendedOutputsSpec, out) {
std::string_view str = "foo^out";
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str);
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = OutputsSpec::Names { "out" };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), str);
}
TEST(ExtendedOutputsSpec, out_bin) {
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bin,out");
}
TEST(ExtendedOutputsSpec, many_carrot) {
auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" };
ASSERT_EQ(extendedOutputsSpec, expected);
ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bar^bin,out");
}
#define TEST_JSON(TYPE, NAME, STR, VAL) \
\
TEST(TYPE, NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
STR ## _json, \
((nlohmann::json) TYPE { VAL })); \
} \
\
TEST(TYPE, NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
TYPE { VAL }, \
(STR ## _json).get<TYPE>()); \
}
TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All { })
TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names { "a" })
TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names { "a", "b" }))
TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default { })
TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit { OutputsSpec::All { } })
TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a" } })
TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a", "b" } }))
#undef TEST_JSON
}

View file

@ -1,46 +0,0 @@
#include "path-with-outputs.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(parseOutputsSpec, basic)
{
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
ASSERT_EQ(prefix, "foo^&*()");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
}
}

144
src/libstore/tests/path.cc Normal file
View file

@ -0,0 +1,144 @@
#include <regex>
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
#include "path-regex.hh"
#include "store-api.hh"
#include "libstoretests.hh"
namespace nix {
#define STORE_DIR "/nix/store/"
#define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q"
class StorePathTest : public LibStoreTest
{
};
static std::regex nameRegex { std::string { nameRegexStr } };
#define TEST_DONT_PARSE(NAME, STR) \
TEST_F(StorePathTest, bad_ ## NAME) { \
std::string_view str = \
STORE_DIR HASH_PART "-" STR; \
ASSERT_THROW( \
store->parseStorePath(str), \
BadStorePath); \
std::string name { STR }; \
EXPECT_FALSE(std::regex_match(name, nameRegex)); \
}
TEST_DONT_PARSE(empty, "")
TEST_DONT_PARSE(garbage, "&*()")
TEST_DONT_PARSE(double_star, "**")
TEST_DONT_PARSE(star_first, "*,foo")
TEST_DONT_PARSE(star_second, "foo,*")
TEST_DONT_PARSE(bang, "foo!o")
#undef TEST_DONT_PARSE
#define TEST_DO_PARSE(NAME, STR) \
TEST_F(StorePathTest, good_ ## NAME) { \
std::string_view str = \
STORE_DIR HASH_PART "-" STR; \
auto p = store->parseStorePath(str); \
std::string name { p.name() }; \
EXPECT_TRUE(std::regex_match(name, nameRegex)); \
}
// 0-9 a-z A-Z + - . _ ? =
TEST_DO_PARSE(numbers, "02345")
TEST_DO_PARSE(lower_case, "foo")
TEST_DO_PARSE(upper_case, "FOO")
TEST_DO_PARSE(plus, "foo+bar")
TEST_DO_PARSE(dash, "foo-dev")
TEST_DO_PARSE(underscore, "foo_bar")
TEST_DO_PARSE(period, "foo.txt")
TEST_DO_PARSE(question_mark, "foo?why")
TEST_DO_PARSE(equals_sign, "foo=foo")
#undef TEST_DO_PARSE
// For rapidcheck
void showValue(const StorePath & p, std::ostream & os) {
os << p.to_string();
}
}
namespace rc {
using namespace nix;
template<>
struct Arbitrary<StorePath> {
static Gen<StorePath> arbitrary();
};
Gen<StorePath> Arbitrary<StorePath>::arbitrary()
{
auto len = *gen::inRange<size_t>(1, StorePath::MaxPathLen);
std::string pre { HASH_PART "-" };
pre.reserve(pre.size() + len);
for (size_t c = 0; c < len; ++c) {
switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) {
case 0 ... 9:
pre += '0' + i;
case 10 ... 35:
pre += 'A' + (i - 10);
break;
case 36 ... 61:
pre += 'a' + (i - 36);
break;
case 62:
pre += '+';
break;
case 63:
pre += '-';
break;
case 64:
pre += '.';
break;
case 65:
pre += '_';
break;
case 66:
pre += '?';
break;
case 67:
pre += '=';
break;
default:
assert(false);
}
}
return gen::just(StorePath { pre });
}
} // namespace rc
namespace nix {
RC_GTEST_FIXTURE_PROP(
StorePathTest,
prop_regex_accept,
(const StorePath & p))
{
RC_ASSERT(std::regex_match(std::string { p.name() }, nameRegex));
}
RC_GTEST_FIXTURE_PROP(
StorePathTest,
prop_round_rip,
(const StorePath & p))
{
RC_ASSERT(p == store->parseStorePath(store->printStorePath(p)));
}
}