1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-24 03:09:35 +01:00

Merge commit '1c1a7074da' into progress-bar

This commit is contained in:
John Ericson 2023-03-11 17:09:17 -05:00
commit d3b5b49ece
34 changed files with 869 additions and 260 deletions

View file

@ -1 +1 @@
2.7.0 2.8.0

View file

@ -72,6 +72,7 @@
- [CLI guideline](contributing/cli-guideline.md) - [CLI guideline](contributing/cli-guideline.md)
- [Release Notes](release-notes/release-notes.md) - [Release Notes](release-notes/release-notes.md)
- [Release X.Y (202?-??-??)](release-notes/rl-next.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
- [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md)
- [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md) - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md)
- [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md)
- [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)

View file

@ -0,0 +1,33 @@
# Release X.Y (2022-03-07)
* Nix will now make some helpful suggestions when you mistype
something on the command line. For instance, if you type `nix build
nixpkgs#thunderbrd`, it will suggest `thunderbird`.
* A number of "default" flake output attributes have been
renamed. These are:
* `defaultPackage.<system>``packages.<system>.default`
* `defaultApps.<system>``apps.<system>.default`
* `defaultTemplate``templates.default`
* `defaultBundler.<system>``bundlers.<system>.default`
* `overlay``overlays.default`
* `devShell.<system>``devShells.<system>.default`
The old flake output attributes still work, but `nix flake check`
will warn about them.
* Breaking API change: `nix bundle` now supports bundlers of the form
`bundler.<system>.<name>= derivation: another-derivation;`. This
supports additional functionality to inspect evaluation information
during bundling. A new
[repository](https://github.com/NixOS/bundlers) has various bundlers
implemented.
* `nix store ping` now reports the version of the remote Nix daemon.
* `nix flake {init,new}` now display information about which files have been
created.
* Templates can now define a `welcomeText` attribute, which is printed out by
`nix flake {init,new} --template <template>`.

View file

@ -1,28 +1 @@
# Release X.Y (202?-??-??) # Release X.Y (202?-??-??)
* A number of "default" flake output attributes have been
renamed. These are:
* `defaultPackage.<system>``packages.<system>.default`
* `defaultApps.<system>``apps.<system>.default`
* `defaultTemplate``templates.default`
* `defaultBundler.<system>``bundlers.<system>.default`
* `overlay``overlays.default`
* `devShell.<system>``devShells.<system>.default`
The old flake output attributes still work, but `nix flake check`
will warn about them.
* `nix bundle` breaking API change now supports bundlers of the form
`bundler.<system>.<name>= derivation: another-derivation;`. This supports
additional functionality to inspect evaluation information during bundling. A
new [repository](https://github.com/NixOS/bundlers) has various bundlers
implemented.
* `nix store ping` now reports the version of the remote Nix daemon.
* `nix flake {init,new}` now display information about which files have been
created.
* Templates can now define a `welcomeText` attribute, which is printed out by
`nix flake {init,new} --template <template>`.

View file

@ -12,6 +12,7 @@
#include "eval-cache.hh" #include "eval-cache.hh"
#include "url.hh" #include "url.hh"
#include "registry.hh" #include "registry.hh"
#include "build-result.hh"
#include <regex> #include <regex>
#include <queue> #include <queue>
@ -272,9 +273,9 @@ void completeFlakeRefWithFragment(
auto attr = root->findAlongAttrPath(attrPath); auto attr = root->findAlongAttrPath(attrPath);
if (!attr) continue; if (!attr) continue;
for (auto & attr2 : attr->getAttrs()) { for (auto & attr2 : (*attr)->getAttrs()) {
if (hasPrefix(attr2, lastAttr)) { if (hasPrefix(attr2, lastAttr)) {
auto attrPath2 = attr->getAttrPath(attr2); auto attrPath2 = (*attr)->getAttrPath(attr2);
/* Strip the attrpath prefix. */ /* Strip the attrpath prefix. */
attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2)); completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
@ -570,15 +571,22 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto cache = openEvalCache(*state, lockedFlake); auto cache = openEvalCache(*state, lockedFlake);
auto root = cache->getRoot(); auto root = cache->getRoot();
Suggestions suggestions;
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
debug("trying flake output attribute '%s'", attrPath); debug("trying flake output attribute '%s'", attrPath);
auto attr = root->findAlongAttrPath( auto attrOrSuggestions = root->findAlongAttrPath(
parseAttrPath(*state, attrPath), parseAttrPath(*state, attrPath),
true true
); );
if (!attr) continue; if (!attrOrSuggestions) {
suggestions += attrOrSuggestions.getSuggestions();
continue;
}
auto attr = *attrOrSuggestions;
if (!attr->isDerivation()) if (!attr->isDerivation())
throw Error("flake output attribute '%s' is not a derivation", attrPath); throw Error("flake output attribute '%s' is not a derivation", attrPath);
@ -593,7 +601,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
} }
throw Error("flake '%s' does not provide attribute %s", throw Error(suggestions, "flake '%s' does not provide attribute %s",
flakeRef, showAttrPaths(getActualAttrPaths())); flakeRef, showAttrPaths(getActualAttrPaths()));
} }
@ -612,17 +620,24 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
auto emptyArgs = state.allocBindings(0); auto emptyArgs = state.allocBindings(0);
Suggestions suggestions;
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
try { try {
auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v, pos); state.forceValue(*v, pos);
return {v, pos}; return {v, pos};
} catch (AttrPathNotFound & e) { } catch (AttrPathNotFound & e) {
suggestions += e.info().suggestions;
} }
} }
throw Error("flake '%s' does not provide attribute %s", throw Error(
flakeRef, showAttrPaths(getActualAttrPaths())); suggestions,
"flake '%s' does not provide attribute %s",
flakeRef,
showAttrPaths(getActualAttrPaths())
);
} }
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
@ -637,7 +652,7 @@ InstallableFlake::getCursors(EvalState & state)
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
if (attr) res.push_back({attr, attrPath}); if (attr) res.push_back({*attr, attrPath});
} }
return res; return res;
@ -757,8 +772,7 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
throw Error( throw Error(
"the derivation '%s' doesn't have an output named '%s'", "the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output); store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled( if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
Xp::CaDerivations)) {
auto outputId = auto outputId =
DrvOutput{outputHashes.at(output), output}; DrvOutput{outputHashes.at(output), output};
auto realisation = auto realisation =
@ -804,12 +818,33 @@ BuiltPaths Installable::build(
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
} }
if (mode == Realise::Nothing || mode == Realise::Derivation) switch (mode) {
case Realise::Nothing:
case Realise::Derivation:
printMissing(store, pathsToBuild, lvlError); printMissing(store, pathsToBuild, lvlError);
else if (mode == Realise::Outputs) return getBuiltPaths(evalStore, store, pathsToBuild);
store->buildPaths(pathsToBuild, bMode, evalStore); case Realise::Outputs: {
BuiltPaths res;
return getBuiltPaths(evalStore, store, pathsToBuild); for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
if (!buildResult.success())
buildResult.rethrow();
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath);
res.push_back(BuiltPath::Built { bfd.drvPath, outputs });
},
[&](const DerivedPath::Opaque & bo) {
res.push_back(BuiltPath::Opaque { bo.path });
},
}, buildResult.path.raw());
}
return res;
}
default:
assert(false);
}
} }
BuiltPaths Installable::toBuiltPaths( BuiltPaths Installable::toBuiltPaths(

View file

@ -74,8 +74,14 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const std::string &
throw Error("empty attribute name in selection path '%1%'", attrPath); throw Error("empty attribute name in selection path '%1%'", attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end()) if (a == v->attrs->end()) {
throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); std::set<std::string> attrNames;
for (auto & attr : *v->attrs)
attrNames.insert(attr.name);
auto suggestions = Suggestions::bestMatches(attrNames, attr);
throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);
}
v = &*a->value; v = &*a->value;
pos = *a->pos; pos = *a->pos;
} }

View file

@ -406,6 +406,16 @@ Value & AttrCursor::forceValue()
return v; return v;
} }
Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
{
auto attrNames = getAttrs();
std::set<std::string> strAttrNames;
for (auto & name : attrNames)
strAttrNames.insert(std::string(name));
return Suggestions::bestMatches(strAttrNames, name);
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
{ {
if (root->db) { if (root->db) {
@ -446,6 +456,11 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
return nullptr; return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr()); //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
for (auto & attr : *v.attrs) {
if (root->db)
root->db->setPlaceholder({cachedValue->first, attr.name});
}
auto attr = v.attrs->get(name); auto attr = v.attrs->get(name);
if (!attr) { if (!attr) {
@ -464,7 +479,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
} }
return std::make_shared<AttrCursor>( return make_ref<AttrCursor>(
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
} }
@ -473,27 +488,31 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
return maybeGetAttr(root->state.symbols.create(name)); return maybeGetAttr(root->state.symbols.create(name));
} }
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors) ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
{ {
auto p = maybeGetAttr(name, forceErrors); auto p = maybeGetAttr(name, forceErrors);
if (!p) if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name)); throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p; return ref(p);
} }
std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name) ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
{ {
return getAttr(root->state.symbols.create(name)); return getAttr(root->state.symbols.create(name));
} }
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force) OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
{ {
auto res = shared_from_this(); auto res = shared_from_this();
for (auto & attr : attrPath) { for (auto & attr : attrPath) {
res = res->maybeGetAttr(attr, force); auto child = res->maybeGetAttr(attr, force);
if (!res) return {}; if (!child) {
auto suggestions = res->getSuggestionsForAttr(attr);
return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
}
res = child;
} }
return res; return ref(res);
} }
std::string AttrCursor::getString() std::string AttrCursor::getString()

View file

@ -94,15 +94,17 @@ public:
std::string getAttrPathStr(Symbol name) const; std::string getAttrPathStr(Symbol name) const;
Suggestions getSuggestionsForAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false); std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name); std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false); ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> getAttr(std::string_view name); ref<AttrCursor> getAttr(std::string_view name);
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
std::string getString(); std::string getString();

View file

@ -63,9 +63,15 @@ static char * dupString(const char * s)
} }
static char * dupStringWithLen(const char * s, size_t size) // When there's no need to write to the string, we can optimize away empty
// string allocations.
// This function handles makeImmutableStringWithLen(null, 0) by returning the
// empty string.
static const char * makeImmutableStringWithLen(const char * s, size_t size)
{ {
char * t; char * t;
if (size == 0)
return "";
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
t = GC_STRNDUP(s, size); t = GC_STRNDUP(s, size);
#else #else
@ -75,6 +81,10 @@ static char * dupStringWithLen(const char * s, size_t size)
return t; return t;
} }
static inline const char * makeImmutableString(std::string_view s) {
return makeImmutableStringWithLen(s.data(), s.size());
}
RootValue allocRootValue(Value * v) RootValue allocRootValue(Value * v)
{ {
@ -805,7 +815,7 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con
void Value::mkString(std::string_view s) void Value::mkString(std::string_view s)
{ {
mkString(dupStringWithLen(s.data(), s.size())); mkString(makeImmutableString(s));
} }
@ -836,7 +846,7 @@ void Value::mkStringMove(const char * s, const PathSet & context)
void Value::mkPath(std::string_view s) void Value::mkPath(std::string_view s)
{ {
mkPath(dupStringWithLen(s.data(), s.size())); mkPath(makeImmutableString(s));
} }

View file

@ -28,6 +28,7 @@ struct BuildResult
LogLimitExceeded, LogLimitExceeded,
NotDeterministic, NotDeterministic,
ResolvesToAlreadyValid, ResolvesToAlreadyValid,
NoSubstituters,
} status = MiscFailure; } status = MiscFailure;
std::string errorMsg; std::string errorMsg;
@ -63,15 +64,26 @@ struct BuildResult
non-determinism.) */ non-determinism.) */
bool isNonDeterministic = false; bool isNonDeterministic = false;
/* The derivation we built or the store path we substituted. */
DerivedPath path;
/* For derivations, the derivation path and the wanted outputs. */
std::optional<StorePath> drvPath;
DrvOutputs builtOutputs; DrvOutputs builtOutputs;
/* The start/stop times of the build (or one of the rounds, if it /* The start/stop times of the build (or one of the rounds, if it
was repeated). */ was repeated). */
time_t startTime = 0, stopTime = 0; time_t startTime = 0, stopTime = 0;
bool success() { bool success()
{
return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;
} }
void rethrow()
{
throw Error("%s", errorMsg);
}
}; };
} }

View file

@ -66,7 +66,7 @@ namespace nix {
DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(true) , useDerivation(true)
, drvPath(drvPath) , drvPath(drvPath)
, wantedOutputs(wantedOutputs) , wantedOutputs(wantedOutputs)
@ -85,7 +85,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(false) , useDerivation(false)
, drvPath(drvPath) , drvPath(drvPath)
, wantedOutputs(wantedOutputs) , wantedOutputs(wantedOutputs)
@ -135,7 +135,7 @@ void DerivationGoal::killChild()
void DerivationGoal::timedOut(Error && ex) void DerivationGoal::timedOut(Error && ex)
{ {
killChild(); killChild();
done(BuildResult::TimedOut, ex); done(BuildResult::TimedOut, {}, ex);
} }
@ -182,7 +182,7 @@ void DerivationGoal::loadDerivation()
trace("loading derivation"); trace("loading derivation");
if (nrFailed != 0) { if (nrFailed != 0) {
done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
return; return;
} }
@ -215,28 +215,20 @@ void DerivationGoal::haveDerivation()
auto outputHashes = staticOutputHashes(worker.evalStore, *drv); auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes) for (auto & [outputName, outputHash] : outputHashes)
initialOutputs.insert({ initialOutputs.insert({
outputName, outputName,
InitialOutput{ InitialOutput {
.wanted = true, // Will be refined later .wanted = true, // Will be refined later
.outputHash = outputHash .outputHash = outputHash
} }
}); });
/* Check what outputs paths are not already valid. */ /* Check what outputs paths are not already valid. */
checkPathValidity(); auto [allValid, validOutputs] = checkPathValidity();
bool allValid = true;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid()) {
allValid = false;
break;
}
}
/* If they are all valid, then we're done. */ /* If they are all valid, then we're done. */
if (allValid && buildMode == bmNormal) { if (allValid && buildMode == bmNormal) {
done(BuildResult::AlreadyValid); done(BuildResult::AlreadyValid, std::move(validOutputs));
return; return;
} }
@ -277,7 +269,7 @@ void DerivationGoal::outputsSubstitutionTried()
trace("all outputs substituted (maybe)"); trace("all outputs substituted (maybe)");
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure, done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
worker.store.printStorePath(drvPath))); worker.store.printStorePath(drvPath)));
return; return;
@ -301,23 +293,17 @@ void DerivationGoal::outputsSubstitutionTried()
return; return;
} }
checkPathValidity(); auto [allValid, validOutputs] = checkPathValidity();
size_t nrInvalid = 0;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid())
nrInvalid++;
}
if (buildMode == bmNormal && nrInvalid == 0) { if (buildMode == bmNormal && allValid) {
done(BuildResult::Substituted); done(BuildResult::Substituted, std::move(validOutputs));
return; return;
} }
if (buildMode == bmRepair && nrInvalid == 0) { if (buildMode == bmRepair && allValid) {
repairClosure(); repairClosure();
return; return;
} }
if (buildMode == bmCheck && nrInvalid > 0) if (buildMode == bmCheck && !allValid)
throw Error("some outputs of '%s' are not valid, so checking is not possible", throw Error("some outputs of '%s' are not valid, so checking is not possible",
worker.store.printStorePath(drvPath)); worker.store.printStorePath(drvPath));
@ -409,7 +395,7 @@ void DerivationGoal::repairClosure()
} }
if (waitees.empty()) { if (waitees.empty()) {
done(BuildResult::AlreadyValid); done(BuildResult::AlreadyValid, assertPathValidity());
return; return;
} }
@ -423,7 +409,7 @@ void DerivationGoal::closureRepaired()
if (nrFailed > 0) if (nrFailed > 0)
throw Error("some paths in the output closure of derivation '%s' could not be repaired", throw Error("some paths in the output closure of derivation '%s' could not be repaired",
worker.store.printStorePath(drvPath)); worker.store.printStorePath(drvPath));
done(BuildResult::AlreadyValid); done(BuildResult::AlreadyValid, assertPathValidity());
} }
@ -434,7 +420,7 @@ void DerivationGoal::inputsRealised()
if (nrFailed != 0) { if (nrFailed != 0) {
if (!useDerivation) if (!useDerivation)
throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
done(BuildResult::DependencyFailed, Error( done(BuildResult::DependencyFailed, {}, Error(
"%s dependencies of derivation '%s' failed to build", "%s dependencies of derivation '%s' failed to build",
nrFailed, worker.store.printStorePath(drvPath))); nrFailed, worker.store.printStorePath(drvPath)));
return; return;
@ -523,10 +509,11 @@ void DerivationGoal::inputsRealised()
state = &DerivationGoal::tryToBuild; state = &DerivationGoal::tryToBuild;
worker.wakeUp(shared_from_this()); worker.wakeUp(shared_from_this());
result = BuildResult(); buildResult = BuildResult { .path = buildResult.path };
} }
void DerivationGoal::started() { void DerivationGoal::started()
{
auto msg = fmt( auto msg = fmt(
buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmRepair ? "repairing outputs of '%s'" :
buildMode == bmCheck ? "checking outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" :
@ -588,19 +575,12 @@ void DerivationGoal::tryToBuild()
omitted, but that would be less efficient.) Note that since we omitted, but that would be less efficient.) Note that since we
now hold the locks on the output paths, no other process can now hold the locks on the output paths, no other process can
build this derivation, so no further checks are necessary. */ build this derivation, so no further checks are necessary. */
checkPathValidity(); auto [allValid, validOutputs] = checkPathValidity();
bool allValid = true;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid()) {
allValid = false;
break;
}
}
if (buildMode != bmCheck && allValid) { if (buildMode != bmCheck && allValid) {
debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
outputLocks.setDeletion(true); outputLocks.setDeletion(true);
done(BuildResult::AlreadyValid); done(BuildResult::AlreadyValid, std::move(validOutputs));
return; return;
} }
@ -626,7 +606,7 @@ void DerivationGoal::tryToBuild()
/* Yes, it has started doing so. Wait until we get /* Yes, it has started doing so. Wait until we get
EOF from the hook. */ EOF from the hook. */
actLock.reset(); actLock.reset();
result.startTime = time(0); // inexact buildResult.startTime = time(0); // inexact
startTime = std::chrono::steady_clock::now(); startTime = std::chrono::steady_clock::now();
state = &DerivationGoal::buildDone; state = &DerivationGoal::buildDone;
started(); started();
@ -831,8 +811,8 @@ void DerivationGoal::buildDone()
debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
result.timesBuilt++; buildResult.timesBuilt++;
result.stopTime = time(0); buildResult.stopTime = time(0);
/* So the child is gone now. */ /* So the child is gone now. */
worker.childTerminated(this); worker.childTerminated(this);
@ -877,11 +857,11 @@ void DerivationGoal::buildDone()
/* Compute the FS closure of the outputs and register them as /* Compute the FS closure of the outputs and register them as
being valid. */ being valid. */
registerOutputs(); auto builtOutputs = registerOutputs();
StorePathSet outputPaths; StorePathSet outputPaths;
for (auto & [_, path] : finalOutputs) for (auto & [_, output] : buildResult.builtOutputs)
outputPaths.insert(path); outputPaths.insert(output.outPath);
runPostBuildHook( runPostBuildHook(
worker.store, worker.store,
*logger, *logger,
@ -891,7 +871,7 @@ void DerivationGoal::buildDone()
if (buildMode == bmCheck) { if (buildMode == bmCheck) {
cleanupPostOutputsRegisteredModeCheck(); cleanupPostOutputsRegisteredModeCheck();
done(BuildResult::Built); done(BuildResult::Built, std::move(builtOutputs));
return; return;
} }
@ -912,6 +892,8 @@ void DerivationGoal::buildDone()
outputLocks.setDeletion(true); outputLocks.setDeletion(true);
outputLocks.unlock(); outputLocks.unlock();
done(BuildResult::Built, std::move(builtOutputs));
} catch (BuildError & e) { } catch (BuildError & e) {
outputLocks.unlock(); outputLocks.unlock();
@ -931,12 +913,10 @@ void DerivationGoal::buildDone()
BuildResult::PermanentFailure; BuildResult::PermanentFailure;
} }
done(st, e); done(st, {}, e);
return; return;
} }
done(BuildResult::Built);
auto stopTime = std::chrono::steady_clock::now(); auto stopTime = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(stopTime - startTime).count() / 1000.0; auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(stopTime - startTime).count() / 1000.0;
@ -948,7 +928,8 @@ void DerivationGoal::buildDone()
duration); duration);
} }
void DerivationGoal::resolvedFinished() { void DerivationGoal::resolvedFinished()
{
assert(resolvedDrvGoal); assert(resolvedDrvGoal);
auto resolvedDrv = *resolvedDrvGoal->drv; auto resolvedDrv = *resolvedDrvGoal->drv;
@ -961,11 +942,13 @@ void DerivationGoal::resolvedFinished() {
if (realWantedOutputs.empty()) if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames(); realWantedOutputs = resolvedDrv.outputNames();
DrvOutputs builtOutputs;
for (auto & wantedOutput : realWantedOutputs) { for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0); assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0); assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = worker.store.queryRealisation( auto realisation = worker.store.queryRealisation(
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
); );
// We've just built it, but maybe the build failed, in which case the // We've just built it, but maybe the build failed, in which case the
// realisation won't be there // realisation won't be there
@ -977,10 +960,11 @@ void DerivationGoal::resolvedFinished() {
signRealisation(newRealisation); signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath); outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation->id, *realisation);
} else { } else {
// If we don't have a realisation, then it must mean that something // If we don't have a realisation, then it must mean that something
// failed when building the resolved drv // failed when building the resolved drv
assert(!result.success()); assert(!buildResult.success());
} }
} }
@ -992,7 +976,7 @@ void DerivationGoal::resolvedFinished() {
); );
auto status = [&]() { auto status = [&]() {
auto resolvedResult = resolvedDrvGoal->getResult(); auto & resolvedResult = resolvedDrvGoal->buildResult;
switch (resolvedResult.status) { switch (resolvedResult.status) {
case BuildResult::AlreadyValid: case BuildResult::AlreadyValid:
return BuildResult::ResolvesToAlreadyValid; return BuildResult::ResolvesToAlreadyValid;
@ -1001,7 +985,7 @@ void DerivationGoal::resolvedFinished() {
} }
}(); }();
done(status); done(status, std::move(builtOutputs));
} }
HookReply DerivationGoal::tryBuildHook() HookReply DerivationGoal::tryBuildHook()
@ -1111,7 +1095,7 @@ HookReply DerivationGoal::tryBuildHook()
} }
void DerivationGoal::registerOutputs() DrvOutputs DerivationGoal::registerOutputs()
{ {
/* When using a build hook, the build hook can register the output /* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have as valid (by doing `nix-store --import'). If so we don't have
@ -1120,21 +1104,7 @@ void DerivationGoal::registerOutputs()
We can only early return when the outputs are known a priori. For We can only early return when the outputs are known a priori. For
floating content-addressed derivations this isn't the case. floating content-addressed derivations this isn't the case.
*/ */
for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { return assertPathValidity();
if (!wantOutput(outputName, wantedOutputs))
continue;
if (!optOutputPath)
throw BuildError(
"output '%s' from derivation '%s' does not have a known output path",
outputName, worker.store.printStorePath(drvPath));
auto & outputPath = *optOutputPath;
if (!worker.store.isValidPath(outputPath))
throw BuildError(
"output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid",
outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath));
finalOutputs.insert_or_assign(outputName, outputPath);
}
} }
Path DerivationGoal::openLogFile() Path DerivationGoal::openLogFile()
@ -1196,7 +1166,7 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
if (settings.maxLogSize && logSize > settings.maxLogSize) { if (settings.maxLogSize && logSize > settings.maxLogSize) {
killChild(); killChild();
done( done(
BuildResult::LogLimitExceeded, BuildResult::LogLimitExceeded, {},
Error("%s killed after writing more than %d bytes of log output", Error("%s killed after writing more than %d bytes of log output",
getName(), settings.maxLogSize)); getName(), settings.maxLogSize));
return; return;
@ -1285,10 +1255,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
} }
void DerivationGoal::checkPathValidity() std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
{ {
bool checkHash = buildMode == bmRepair; bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs; auto wantedOutputsLeft = wantedOutputs;
DrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) { for (auto & i : queryPartialDerivationOutputMap()) {
InitialOutput & info = initialOutputs.at(i.first); InitialOutput & info = initialOutputs.at(i.first);
info.wanted = wantOutput(i.first, wantedOutputs); info.wanted = wantOutput(i.first, wantedOutputs);
@ -1305,26 +1277,28 @@ void DerivationGoal::checkPathValidity()
: PathStatus::Corrupt, : PathStatus::Corrupt,
}; };
} }
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
if (auto real = worker.store.queryRealisation(drvOutput)) { if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = { info.known = {
.path = real->outPath, .path = real->outPath,
.status = PathStatus::Valid, .status = PathStatus::Valid,
}; };
} else if (info.known && info.known->status == PathStatus::Valid) { } else if (info.known && info.known->isValid()) {
// We know the output because it' a static output of the // We know the output because it's a static output of the
// derivation, and the output path is valid, but we don't have // derivation, and the output path is valid, but we don't have
// its realisation stored (probably because it has been built // its realisation stored (probably because it has been built
// without the `ca-derivations` experimental flag) // without the `ca-derivations` experimental flag).
worker.store.registerDrvOutput( worker.store.registerDrvOutput(
Realisation{ Realisation {
drvOutput, drvOutput,
info.known->path, info.known->path,
} }
); );
} }
} }
if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
} }
// If we requested all the outputs via the empty set, we are always fine. // If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid // If we requested specific elements, the loop above removes all the valid
@ -1333,24 +1307,51 @@ void DerivationGoal::checkPathValidity()
throw Error("derivation '%s' does not have wanted outputs %s", throw Error("derivation '%s' does not have wanted outputs %s",
worker.store.printStorePath(drvPath), worker.store.printStorePath(drvPath),
concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
bool allValid = true;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid()) {
allValid = false;
break;
}
}
return { allValid, validOutputs };
} }
void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) DrvOutputs DerivationGoal::assertPathValidity()
{ {
result.status = status; auto [allValid, validOutputs] = checkPathValidity();
if (!allValid)
throw Error("some outputs are unexpectedly invalid");
return validOutputs;
}
void DerivationGoal::done(
BuildResult::Status status,
DrvOutputs builtOutputs,
std::optional<Error> ex)
{
buildResult.drvPath = drvPath;
buildResult.status = status;
if (ex) if (ex)
result.errorMsg = ex->what(); // FIXME: strip: "error: "
amDone(result.success() ? ecSuccess : ecFailed, ex); buildResult.errorMsg = ex->what();
if (result.status == BuildResult::TimedOut) amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true; worker.timedOut = true;
if (result.status == BuildResult::PermanentFailure) if (buildResult.status == BuildResult::PermanentFailure)
worker.permanentFailure = true; worker.permanentFailure = true;
mcExpectedBuilds.reset(); mcExpectedBuilds.reset();
mcRunningBuilds.reset(); mcRunningBuilds.reset();
if (result.success()) { if (buildResult.success()) {
assert(!builtOutputs.empty());
buildResult.builtOutputs = std::move(builtOutputs);
if (status == BuildResult::Built) if (status == BuildResult::Built)
worker.doneBuilds++; worker.doneBuilds++;
} else { } else {
@ -1364,7 +1365,7 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
if (traceBuiltOutputsFile != "") { if (traceBuiltOutputsFile != "") {
std::fstream fs; std::fstream fs;
fs.open(traceBuiltOutputsFile, std::fstream::out); fs.open(traceBuiltOutputsFile, std::fstream::out);
fs << worker.store.printStorePath(drvPath) << "\t" << result.toString() << std::endl; fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
} }
} }

View file

@ -2,7 +2,6 @@
#include "parsed-derivations.hh" #include "parsed-derivations.hh"
#include "lock.hh" #include "lock.hh"
#include "build-result.hh"
#include "store-api.hh" #include "store-api.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "goal.hh" #include "goal.hh"
@ -105,20 +104,8 @@ struct DerivationGoal : public Goal
typedef void (DerivationGoal::*GoalState)(); typedef void (DerivationGoal::*GoalState)();
GoalState state; GoalState state;
/* The final output paths of the build.
- For input-addressed derivations, always the precomputed paths
- For content-addressed derivations, calcuated from whatever the hash
ends up being. (Note that fixed outputs derivations that produce the
"wrong" output still install that data under its true content-address.)
*/
OutputPathMap finalOutputs;
BuildMode buildMode; BuildMode buildMode;
BuildResult result;
/* Time the build started. 'result' also has a 'startTime' field, /* Time the build started. 'result' also has a 'startTime' field,
but that's wall clock time, so we can't use it to compute the but that's wall clock time, so we can't use it to compute the
build duration... */ build duration... */
@ -158,8 +145,6 @@ struct DerivationGoal : public Goal
/* Add wanted outputs to an already existing derivation goal. */ /* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs); void addWantedOutputs(const StringSet & outputs);
BuildResult getResult() { return result; }
/* The states. */ /* The states. */
void getDerivation(); void getDerivation();
void loadDerivation(); void loadDerivation();
@ -181,7 +166,7 @@ struct DerivationGoal : public Goal
/* Check that the derivation outputs all exist and register them /* Check that the derivation outputs all exist and register them
as valid. */ as valid. */
virtual void registerOutputs(); virtual DrvOutputs registerOutputs();
/* Open a log file and a pipe to it. */ /* Open a log file and a pipe to it. */
Path openLogFile(); Path openLogFile();
@ -216,8 +201,17 @@ struct DerivationGoal : public Goal
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap(); OutputPathMap queryDerivationOutputMap();
/* Return the set of (in)valid paths. */ /* Update 'initialOutputs' to determine the current status of the
void checkPathValidity(); outputs of the derivation. Also returns a Boolean denoting
whether all outputs are valid and non-corrupt, and a
'DrvOutputs' structure containing the valid and wanted
outputs. */
std::pair<bool, DrvOutputs> checkPathValidity();
/* Aborts if any output is not valid or corrupt, and otherwise
returns a 'DrvOutputs' structure containing the wanted
outputs. */
DrvOutputs assertPathValidity();
/* Forcibly kill the child process, if any. */ /* Forcibly kill the child process, if any. */
virtual void killChild(); virtual void killChild();
@ -228,6 +222,7 @@ struct DerivationGoal : public Goal
void done( void done(
BuildResult::Status status, BuildResult::Status status,
DrvOutputs builtOutputs = {},
std::optional<Error> ex = {}); std::optional<Error> ex = {});
StorePathSet exportReferences(const StorePathSet & storePaths); StorePathSet exportReferences(const StorePathSet & storePaths);

View file

@ -6,8 +6,12 @@
namespace nix { namespace nix {
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
: Goal(worker) const DrvOutput & id,
Worker & worker,
RepairFlag repair,
std::optional<ContentAddress> ca)
: Goal(worker, DerivedPath::Opaque { StorePath::dummy })
, id(id) , id(id)
{ {
state = &DrvOutputSubstitutionGoal::init; state = &DrvOutputSubstitutionGoal::init;
@ -32,7 +36,7 @@ void DrvOutputSubstitutionGoal::init()
void DrvOutputSubstitutionGoal::tryNext() void DrvOutputSubstitutionGoal::tryNext()
{ {
trace("Trying next substituter"); trace("trying next substituter");
if (subs.size() == 0) { if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal /* None left. Terminate this goal and let someone else deal
@ -119,7 +123,7 @@ void DrvOutputSubstitutionGoal::realisationFetched()
void DrvOutputSubstitutionGoal::outPathValid() void DrvOutputSubstitutionGoal::outPathValid()
{ {
assert(outputInfo); assert(outputInfo);
trace("Output path substituted"); trace("output path substituted");
if (nrFailed > 0) { if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); debug("The output path of the derivation output '%s' could not be substituted", id.to_string());

View file

@ -47,43 +47,51 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
} }
} }
std::vector<BuildResult> Store::buildPathsWithResults(
const std::vector<DerivedPath> & reqs,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
{
Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals;
for (const auto & br : reqs) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](const DerivedPath::Opaque & bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
}
worker.run(goals);
std::vector<BuildResult> results;
for (auto & i : goals)
results.push_back(i->buildResult);
return results;
}
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) BuildMode buildMode)
{ {
Worker worker(*this, *this); Worker worker(*this, *this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
BuildResult result;
try { try {
worker.run(Goals{goal}); worker.run(Goals{goal});
result = goal->getResult(); return goal->buildResult;
} catch (Error & e) { } catch (Error & e) {
result.status = BuildResult::MiscFailure; return BuildResult {
result.errorMsg = e.msg(); .status = BuildResult::MiscFailure,
} .errorMsg = e.msg(),
// XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's .path = DerivedPath::Built { .drvPath = drvPath },
// extended to return the full realisation for each output };
auto staticDrvOutputs = drv.outputsAndOptPaths(*this); };
auto outputHashes = staticOutputHashes(*this, drv);
for (auto & [outputName, staticOutput] : staticDrvOutputs) {
auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
if (staticOutput.second)
result.builtOutputs.insert_or_assign(
outputId,
Realisation{ outputId, *staticOutput.second}
);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
auto realisation = this->queryRealisation(outputId);
if (realisation)
result.builtOutputs.insert_or_assign(
outputId,
*realisation
);
}
}
return result;
} }

View file

@ -2,6 +2,7 @@
#include "types.hh" #include "types.hh"
#include "store-api.hh" #include "store-api.hh"
#include "build-result.hh"
namespace nix { namespace nix {
@ -55,10 +56,15 @@ struct Goal : public std::enable_shared_from_this<Goal>
/* Whether the goal is finished. */ /* Whether the goal is finished. */
ExitCode exitCode; ExitCode exitCode;
/* Build result. */
BuildResult buildResult;
/* Exception containing an error message, if any. */ /* Exception containing an error message, if any. */
std::optional<Error> ex; std::optional<Error> ex;
Goal(Worker & worker) : worker(worker) Goal(Worker & worker, DerivedPath path)
: worker(worker)
, buildResult { .path = std::move(path) }
{ {
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
exitCode = ecBusy; exitCode = ecBusy;

View file

@ -194,7 +194,7 @@ void LocalDerivationGoal::tryLocalBuild() {
outputLocks.unlock(); outputLocks.unlock();
buildUser.reset(); buildUser.reset();
worker.permanentFailure = true; worker.permanentFailure = true;
done(BuildResult::InputRejected, e); done(BuildResult::InputRejected, {}, e);
return; return;
} }
@ -756,7 +756,7 @@ void LocalDerivationGoal::startBuilder()
if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
throw SysError("putting pseudoterminal into raw mode"); throw SysError("putting pseudoterminal into raw mode");
result.startTime = time(0); buildResult.startTime = time(0);
startTime = std::chrono::steady_clock::now(); startTime = std::chrono::steady_clock::now();
/* Fork a child to build the package. */ /* Fork a child to build the package. */
@ -1261,6 +1261,16 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
} }
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
for (auto & result : buildPathsWithResults(paths, buildMode, evalStore))
if (!result.success())
result.rethrow();
}
std::vector<BuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr) override
{ {
assert(!evalStore); assert(!evalStore);
@ -1274,26 +1284,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
} }
next->buildPaths(paths, buildMode); auto results = next->buildPathsWithResults(paths, buildMode);
for (auto & path : paths) { for (auto & result : results) {
auto p = std::get_if<DerivedPath::Built>(&path); for (auto & [outputName, output] : result.builtOutputs) {
if (!p) continue; newPaths.insert(output.outPath);
auto & bfd = *p; newRealisations.insert(output);
auto drv = readDerivation(bfd.drvPath); }
auto drvHashes = staticOutputHashes(*this, drv);
auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs)) {
newPaths.insert(outputPath);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto thisRealisation = next->queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName}
);
assert(thisRealisation);
newRealisations.insert(*thisRealisation);
}
}
} }
StorePathSet closure; StorePathSet closure;
@ -1302,6 +1299,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
goal.addDependency(path); goal.addDependency(path);
for (auto & real : Realisation::closure(*next, newRealisations)) for (auto & real : Realisation::closure(*next, newRealisations))
goal.addedDrvOutputs.insert(real.id); goal.addedDrvOutputs.insert(real.id);
return results;
} }
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@ -2070,7 +2069,7 @@ void LocalDerivationGoal::runChild()
} }
void LocalDerivationGoal::registerOutputs() DrvOutputs LocalDerivationGoal::registerOutputs()
{ {
/* When using a build hook, the build hook can register the output /* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have as valid (by doing `nix-store --import'). If so we don't have
@ -2079,10 +2078,8 @@ void LocalDerivationGoal::registerOutputs()
We can only early return when the outputs are known a priori. For We can only early return when the outputs are known a priori. For
floating content-addressed derivations this isn't the case. floating content-addressed derivations this isn't the case.
*/ */
if (hook) { if (hook)
DerivationGoal::registerOutputs(); return DerivationGoal::registerOutputs();
return;
}
std::map<std::string, ValidPathInfo> infos; std::map<std::string, ValidPathInfo> infos;
@ -2205,6 +2202,8 @@ void LocalDerivationGoal::registerOutputs()
std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
OutputPathMap finalOutputs;
for (auto & outputName : sortedOutputNames) { for (auto & outputName : sortedOutputNames) {
auto output = drv->outputs.at(outputName); auto output = drv->outputs.at(outputName);
auto & scratchPath = scratchOutputs.at(outputName); auto & scratchPath = scratchOutputs.at(outputName);
@ -2341,6 +2340,7 @@ void LocalDerivationGoal::registerOutputs()
}; };
ValidPathInfo newInfo = std::visit(overloaded { ValidPathInfo newInfo = std::visit(overloaded {
[&](const DerivationOutputInputAddressed & output) { [&](const DerivationOutputInputAddressed & output) {
/* input-addressed case */ /* input-addressed case */
auto requiredFinalPath = output.path; auto requiredFinalPath = output.path;
@ -2360,6 +2360,7 @@ void LocalDerivationGoal::registerOutputs()
newInfo0.references.insert(newInfo0.path); newInfo0.references.insert(newInfo0.path);
return newInfo0; return newInfo0;
}, },
[&](const DerivationOutputCAFixed & dof) { [&](const DerivationOutputCAFixed & dof) {
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
.method = dof.hash.method, .method = dof.hash.method,
@ -2382,14 +2383,17 @@ void LocalDerivationGoal::registerOutputs()
} }
return newInfo0; return newInfo0;
}, },
[&](DerivationOutputCAFloating dof) {
[&](DerivationOutputCAFloating & dof) {
return newInfoFromCA(dof); return newInfoFromCA(dof);
}, },
[&](DerivationOutputDeferred) -> ValidPathInfo { [&](DerivationOutputDeferred) -> ValidPathInfo {
// No derivation should reach that point without having been // No derivation should reach that point without having been
// rewritten first // rewritten first
assert(false); assert(false);
}, },
}, output.output); }, output.output);
/* FIXME: set proper permissions in restorePath() so /* FIXME: set proper permissions in restorePath() so
@ -2500,11 +2504,12 @@ void LocalDerivationGoal::registerOutputs()
} }
if (buildMode == bmCheck) { if (buildMode == bmCheck) {
// In case of FOD mismatches on `--check` an error must be thrown as this is also /* In case of fixed-output derivations, if there are
// a source for non-determinism. mismatches on `--check` an error must be thrown as this is
also a source for non-determinism. */
if (delayedException) if (delayedException)
std::rethrow_exception(delayedException); std::rethrow_exception(delayedException);
return; return assertPathValidity();
} }
/* Apply output checks. */ /* Apply output checks. */
@ -2516,7 +2521,7 @@ void LocalDerivationGoal::registerOutputs()
assert(prevInfos.size() == infos.size()); assert(prevInfos.size() == infos.size());
for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
if (!(*i == *j)) { if (!(*i == *j)) {
result.isNonDeterministic = true; buildResult.isNonDeterministic = true;
Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
bool prevExists = keepPreviousRound && pathExists(prev); bool prevExists = keepPreviousRound && pathExists(prev);
hintformat hint = prevExists hintformat hint = prevExists
@ -2554,7 +2559,7 @@ void LocalDerivationGoal::registerOutputs()
if (curRound < nrRounds) { if (curRound < nrRounds) {
prevInfos = std::move(infos); prevInfos = std::move(infos);
return; return {};
} }
/* Remove the .check directories if we're done. FIXME: keep them /* Remove the .check directories if we're done. FIXME: keep them
@ -2589,17 +2594,24 @@ void LocalDerivationGoal::registerOutputs()
means it's safe to link the derivation to the output hash. We must do means it's safe to link the derivation to the output hash. We must do
that for floating CA derivations, which otherwise couldn't be cached, that for floating CA derivations, which otherwise couldn't be cached,
but it's fine to do in all cases. */ but it's fine to do in all cases. */
DrvOutputs builtOutputs;
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { for (auto & [outputName, newInfo] : infos) {
for (auto& [outputName, newInfo] : infos) { auto thisRealisation = Realisation {
auto thisRealisation = Realisation{ .id = DrvOutput {
.id = DrvOutput{initialOutputs.at(outputName).outputHash, initialOutputs.at(outputName).outputHash,
outputName}, outputName
.outPath = newInfo.path}; },
.outPath = newInfo.path
};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
signRealisation(thisRealisation); signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation); worker.store.registerDrvOutput(thisRealisation);
} }
builtOutputs.emplace(thisRealisation.id, thisRealisation);
} }
return builtOutputs;
} }
void LocalDerivationGoal::signRealisation(Realisation & realisation) void LocalDerivationGoal::signRealisation(Realisation & realisation)
@ -2608,7 +2620,7 @@ void LocalDerivationGoal::signRealisation(Realisation & realisation)
} }
void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & outputs)
{ {
std::map<Path, const ValidPathInfo &> outputsByPath; std::map<Path, const ValidPathInfo &> outputsByPath;
for (auto & output : outputs) for (auto & output : outputs)
@ -2680,8 +2692,8 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
for (auto & i : *value) { for (auto & i : *value) {
if (worker.store.isStorePath(i)) if (worker.store.isStorePath(i))
spec.insert(worker.store.parseStorePath(i)); spec.insert(worker.store.parseStorePath(i));
else if (finalOutputs.count(i)) else if (outputs.count(i))
spec.insert(finalOutputs.at(i)); spec.insert(outputs.at(i).path);
else throw BuildError("derivation contains an illegal reference specifier '%s'", i); else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
} }

View file

@ -169,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal
/* Check that the derivation outputs all exist and register them /* Check that the derivation outputs all exist and register them
as valid. */ as valid. */
void registerOutputs() override; DrvOutputs registerOutputs() override;
void signRealisation(Realisation &) override; void signRealisation(Realisation &) override;

View file

@ -6,7 +6,7 @@
namespace nix { namespace nix {
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker) : Goal(worker, DerivedPath::Opaque { storePath })
, storePath(storePath) , storePath(storePath)
, repair(repair) , repair(repair)
, ca(ca) , ca(ca)
@ -24,6 +24,13 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
} }
void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status)
{
buildResult.status = status;
amDone(result);
}
void PathSubstitutionGoal::work() void PathSubstitutionGoal::work()
{ {
(this->*state)(); (this->*state)();
@ -38,7 +45,7 @@ void PathSubstitutionGoal::init()
/* If the path already exists we're done. */ /* If the path already exists we're done. */
if (!repair && worker.store.isValidPath(storePath)) { if (!repair && worker.store.isValidPath(storePath)) {
amDone(ecSuccess); done(ecSuccess, BuildResult::AlreadyValid);
return; return;
} }
@ -65,7 +72,7 @@ void PathSubstitutionGoal::tryNext()
/* Hack: don't indicate failure if there were no substituters. /* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a In that case the calling derivation should just do a
build. */ build. */
amDone(substituterFailed ? ecFailed : ecNoSubstituters); done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters);
if (substituterFailed) { if (substituterFailed) {
worker.failedSubstitutions++; worker.failedSubstitutions++;
@ -163,7 +170,9 @@ void PathSubstitutionGoal::referencesValid()
if (nrFailed > 0) { if (nrFailed > 0) {
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); done(
nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
BuildResult::DependencyFailed);
return; return;
} }
@ -270,7 +279,7 @@ void PathSubstitutionGoal::finished()
worker.updateProgress(); worker.updateProgress();
amDone(ecSuccess); done(ecSuccess, BuildResult::Substituted);
auto stopTime = std::chrono::steady_clock::now(); auto stopTime = std::chrono::steady_clock::now();

View file

@ -56,6 +56,8 @@ struct PathSubstitutionGoal : public Goal
/* Time substitution started. */ /* Time substitution started. */
std::chrono::time_point<std::chrono::steady_clock> startTime; std::chrono::time_point<std::chrono::steady_clock> startTime;
void done(ExitCode result, BuildResult::Status status);
public: public:
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~PathSubstitutionGoal(); ~PathSubstitutionGoal();

View file

@ -532,6 +532,25 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break; break;
} }
case wopBuildPathsWithResults: {
auto drvs = readDerivedPaths(*store, clientVersion, from);
BuildMode mode = bmNormal;
mode = (BuildMode) readInt(from);
/* Repairing is not atomic, so disallowed for "untrusted"
clients. */
if (mode == bmRepair && !trusted)
throw Error("repairing is not allowed because you are not in 'trusted-users'");
logger->startWork();
auto results = store->buildPathsWithResults(drvs, mode);
logger->stopWork();
worker_proto::write(*store, to, results);
break;
}
case wopBuildDerivation: { case wopBuildDerivation: {
auto drvPath = store->parseStorePath(readString(from)); auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv; BasicDerivation drv;

View file

@ -279,7 +279,7 @@ public:
conn->to.flush(); conn->to.flush();
BuildResult status; BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } };
status.status = (BuildResult::Status) readInt(conn->from); status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg; conn->from >> status.errorMsg;
@ -317,7 +317,7 @@ public:
conn->to.flush(); conn->to.flush();
BuildResult result; BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } };
result.status = (BuildResult::Status) readInt(conn->from); result.status = (BuildResult::Status) readInt(conn->from);
if (!result.success()) { if (!result.success()) {

View file

@ -91,6 +91,35 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
} }
BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
{
auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
BuildResult res { .path = path };
res.status = (BuildResult::Status) readInt(from);
from
>> res.errorMsg
>> res.timesBuilt
>> res.isNonDeterministic
>> res.startTime
>> res.stopTime;
res.builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
return res;
}
void write(const Store & store, Sink & to, const BuildResult & res)
{
worker_proto::write(store, to, res.path);
to
<< res.status
<< res.errorMsg
<< res.timesBuilt
<< res.isNonDeterministic
<< res.startTime
<< res.stopTime;
worker_proto::write(store, to, res.builtOutputs);
}
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _) std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{ {
auto s = readString(from); auto s = readString(from);
@ -747,17 +776,24 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons
} }
} }
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) void RemoteStore::copyDrvsFromEvalStore(
const std::vector<DerivedPath> & paths,
std::shared_ptr<Store> evalStore)
{ {
if (evalStore && evalStore.get() != this) { if (evalStore && evalStore.get() != this) {
/* The remote doesn't have a way to access evalStore, so copy /* The remote doesn't have a way to access evalStore, so copy
the .drvs. */ the .drvs. */
RealisedPath::Set drvPaths2; RealisedPath::Set drvPaths2;
for (auto & i : drvPaths) for (auto & i : paths)
if (auto p = std::get_if<DerivedPath::Built>(&i)) if (auto p = std::get_if<DerivedPath::Built>(&i))
drvPaths2.insert(p->drvPath); drvPaths2.insert(p->drvPath);
copyClosure(*evalStore, *this, drvPaths2); copyClosure(*evalStore, *this, drvPaths2);
} }
}
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
copyDrvsFromEvalStore(drvPaths, evalStore);
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopBuildPaths; conn->to << wopBuildPaths;
@ -774,6 +810,91 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
readInt(conn->from); readInt(conn->from);
} }
std::vector<BuildResult> RemoteStore::buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
{
copyDrvsFromEvalStore(paths, evalStore);
std::optional<ConnectionHandle> conn_(getConnection());
auto & conn = *conn_;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) {
conn->to << wopBuildPathsWithResults;
writeDerivedPaths(*this, conn, paths);
conn->to << buildMode;
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<std::vector<BuildResult>> {});
} else {
// Avoid deadlock.
conn_.reset();
// Note: this throws an exception if a build/substitution
// fails, but meh.
buildPaths(paths, buildMode, evalStore);
std::vector<BuildResult> results;
for (auto & path : paths) {
std::visit(
overloaded {
[&](const DerivedPath::Opaque & bo) {
results.push_back(BuildResult {
.status = BuildResult::Substituted,
.path = bo,
});
},
[&](const DerivedPath::Built & bfd) {
BuildResult res {
.status = BuildResult::Built,
.path = bfd,
};
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*this);
for (auto & output : bfd.outputs) {
if (!outputHashes.count(output))
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
printStorePath(bfd.drvPath), output);
auto outputId =
DrvOutput{outputHashes.at(output), output};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto realisation =
queryRealisation(outputId);
if (!realisation)
throw Error(
"cannot operate on an output of unbuilt "
"content-addressed derivation '%s'",
outputId.to_string());
res.builtOutputs.emplace(realisation->id, *realisation);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
assert(drvOutputs.count(output));
assert(drvOutputs.at(output).second);
res.builtOutputs.emplace(
outputId,
Realisation {
.id = outputId,
.outPath = *drvOutputs.at(output).second
});
}
}
results.push_back(res);
}
},
path.raw());
}
return results;
}
}
BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) BuildMode buildMode)
@ -783,7 +904,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
writeDerivation(conn->to, *this, drv); writeDerivation(conn->to, *this, drv);
conn->to << buildMode; conn->to << buildMode;
conn.processStderr(); conn.processStderr();
BuildResult res; BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } };
res.status = (BuildResult::Status) readInt(conn->from); res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg; conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {

View file

@ -97,6 +97,11 @@ public:
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
std::vector<BuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override; BuildMode buildMode) override;
@ -171,6 +176,9 @@ private:
std::atomic_bool failed{false}; std::atomic_bool failed{false};
void copyDrvsFromEvalStore(
const std::vector<DerivedPath> & paths,
std::shared_ptr<Store> evalStore);
}; };

View file

@ -432,6 +432,16 @@ public:
BuildMode buildMode = bmNormal, BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr); std::shared_ptr<Store> evalStore = nullptr);
/* Like `buildPaths()`, but return a vector of `BuildResult`s
corresponding to each element in `paths`. Note that in case of
a build/substitution error, this function won't throw an
exception, but return a `BuildResult` containing an error
message. */
virtual std::vector<BuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr);
/* Build a single non-materialized derivation (i.e. not from an /* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file). on-disk .drv file).

View file

@ -9,7 +9,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION (1 << 8 | 33) #define PROTOCOL_VERSION (1 << 8 | 34)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -57,6 +57,7 @@ typedef enum {
wopQueryRealisation = 43, wopQueryRealisation = 43,
wopAddMultipleToStore = 44, wopAddMultipleToStore = 44,
wopAddBuildLog = 45, wopAddBuildLog = 45,
wopBuildPathsWithResults = 46,
} WorkerOp; } WorkerOp;
@ -91,6 +92,7 @@ MAKE_WORKER_PROTO(, ContentAddress);
MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, DerivedPath);
MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(, BuildResult);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>); MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>); MAKE_WORKER_PROTO(template<typename T>, std::set<T>);

View file

@ -328,8 +328,13 @@ MultiCommand::MultiCommand(const Commands & commands_)
completions->add(name); completions->add(name);
} }
auto i = commands.find(s); auto i = commands.find(s);
if (i == commands.end()) if (i == commands.end()) {
throw UsageError("'%s' is not a recognised command", s); std::set<std::string> commandNames;
for (auto & [name, _] : commands)
commandNames.insert(name);
auto suggestions = Suggestions::bestMatches(commandNames, s);
throw UsageError(suggestions, "'%s' is not a recognised command", s);
}
command = {s, i->second()}; command = {s, i->second()};
command->second->parent = this; command->second->parent = this;
}} }}

View file

@ -282,6 +282,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
} }
} }
auto suggestions = einfo.suggestions.trim();
if (! suggestions.suggestions.empty()){
oss << "Did you mean " <<
suggestions.trim() <<
"?" << std::endl;
}
// traces // traces
if (showTrace && !einfo.traces.empty()) { if (showTrace && !einfo.traces.empty()) {
for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) {

View file

@ -1,5 +1,6 @@
#pragma once #pragma once
#include "suggestions.hh"
#include "ref.hh" #include "ref.hh"
#include "types.hh" #include "types.hh"
#include "fmt.hh" #include "fmt.hh"
@ -112,6 +113,8 @@ struct ErrorInfo {
std::optional<ErrPos> errPos; std::optional<ErrPos> errPos;
std::list<Trace> traces; std::list<Trace> traces;
Suggestions suggestions;
static std::optional<std::string> programName; static std::optional<std::string> programName;
}; };
@ -141,6 +144,11 @@ public:
: err { .level = lvlError, .msg = hintfmt(fs, args...) } : err { .level = lvlError, .msg = hintfmt(fs, args...) }
{ } { }
template<typename... Args>
BaseError(const Suggestions & sug, const Args & ... args)
: err { .level = lvlError, .msg = hintfmt(args...), .suggestions = sug }
{ }
BaseError(hintformat hint) BaseError(hintformat hint)
: err { .level = lvlError, .msg = hint } : err { .level = lvlError, .msg = hint }
{ } { }

114
src/libutil/suggestions.cc Normal file
View file

@ -0,0 +1,114 @@
#include "suggestions.hh"
#include "ansicolor.hh"
#include "util.hh"
#include <algorithm>
namespace nix {
int levenshteinDistance(std::string_view first, std::string_view second)
{
// Implementation borrowed from
// https://en.wikipedia.org/wiki/Levenshtein_distance#Iterative_with_two_matrix_rows
int m = first.size();
int n = second.size();
auto v0 = std::vector<int>(n+1);
auto v1 = std::vector<int>(n+1);
for (auto i = 0; i <= n; i++)
v0[i] = i;
for (auto i = 0; i < m; i++) {
v1[0] = i+1;
for (auto j = 0; j < n; j++) {
auto deletionCost = v0[j+1] + 1;
auto insertionCost = v1[j] + 1;
auto substitutionCost = first[i] == second[j] ? v0[j] : v0[j] + 1;
v1[j+1] = std::min({deletionCost, insertionCost, substitutionCost});
}
std::swap(v0, v1);
}
return v0[n];
}
Suggestions Suggestions::bestMatches (
std::set<std::string> allMatches,
std::string query)
{
std::set<Suggestion> res;
for (const auto & possibleMatch : allMatches) {
res.insert(Suggestion {
.distance = levenshteinDistance(query, possibleMatch),
.suggestion = possibleMatch,
});
}
return Suggestions { res };
}
Suggestions Suggestions::trim(int limit, int maxDistance) const
{
std::set<Suggestion> res;
int count = 0;
for (auto & elt : suggestions) {
if (count >= limit || elt.distance > maxDistance)
break;
count++;
res.insert(elt);
}
return Suggestions{res};
}
std::string Suggestion::to_string() const
{
return ANSI_WARNING + filterANSIEscapes(suggestion) + ANSI_NORMAL;
}
std::string Suggestions::to_string() const
{
switch (suggestions.size()) {
case 0:
return "";
case 1:
return suggestions.begin()->to_string();
default: {
std::string res = "one of ";
auto iter = suggestions.begin();
res += iter->to_string(); // Iter cant be end() because the container isnt null
iter++;
auto last = suggestions.end(); last--;
for ( ; iter != suggestions.end() ; iter++) {
res += (iter == last) ? " or " : ", ";
res += iter->to_string();
}
return res;
}
}
}
Suggestions & Suggestions::operator+=(const Suggestions & other)
{
suggestions.insert(
other.suggestions.begin(),
other.suggestions.end()
);
return *this;
}
std::ostream & operator<<(std::ostream & str, const Suggestion & suggestion)
{
return str << suggestion.to_string();
}
std::ostream & operator<<(std::ostream & str, const Suggestions & suggestions)
{
return str << suggestions.to_string();
}
}

102
src/libutil/suggestions.hh Normal file
View file

@ -0,0 +1,102 @@
#pragma once
#include "comparator.hh"
#include "types.hh"
#include <set>
namespace nix {
int levenshteinDistance(std::string_view first, std::string_view second);
/**
* A potential suggestion for the cli interface.
*/
class Suggestion {
public:
int distance; // The smaller the better
std::string suggestion;
std::string to_string() const;
GENERATE_CMP(Suggestion, me->distance, me->suggestion)
};
class Suggestions {
public:
std::set<Suggestion> suggestions;
std::string to_string() const;
Suggestions trim(
int limit = 5,
int maxDistance = 2
) const;
static Suggestions bestMatches (
std::set<std::string> allMatches,
std::string query
);
Suggestions& operator+=(const Suggestions & other);
};
std::ostream & operator<<(std::ostream & str, const Suggestion &);
std::ostream & operator<<(std::ostream & str, const Suggestions &);
// Either a value of type `T`, or some suggestions
template<typename T>
class OrSuggestions {
public:
using Raw = std::variant<T, Suggestions>;
Raw raw;
T* operator ->()
{
return &**this;
}
T& operator *()
{
return std::get<T>(raw);
}
operator bool() const noexcept
{
return std::holds_alternative<T>(raw);
}
OrSuggestions(T t)
: raw(t)
{
}
OrSuggestions()
: raw(Suggestions{})
{
}
static OrSuggestions<T> failed(const Suggestions & s)
{
auto res = OrSuggestions<T>();
res.raw = s;
return res;
}
static OrSuggestions<T> failed()
{
return OrSuggestions<T>::failed(Suggestions{});
}
const Suggestions & getSuggestions()
{
static Suggestions noSuggestions;
if (const auto & suggestions = std::get_if<Suggestions>(&raw))
return *suggestions;
else
return noSuggestions;
}
};
}

View file

@ -0,0 +1,43 @@
#include "suggestions.hh"
#include <gtest/gtest.h>
namespace nix {
struct LevenshteinDistanceParam {
std::string s1, s2;
int distance;
};
class LevenshteinDistanceTest :
public testing::TestWithParam<LevenshteinDistanceParam> {
};
TEST_P(LevenshteinDistanceTest, CorrectlyComputed) {
auto params = GetParam();
ASSERT_EQ(levenshteinDistance(params.s1, params.s2), params.distance);
ASSERT_EQ(levenshteinDistance(params.s2, params.s1), params.distance);
}
INSTANTIATE_TEST_SUITE_P(LevenshteinDistance, LevenshteinDistanceTest,
testing::Values(
LevenshteinDistanceParam{"foo", "foo", 0},
LevenshteinDistanceParam{"foo", "", 3},
LevenshteinDistanceParam{"", "", 0},
LevenshteinDistanceParam{"foo", "fo", 1},
LevenshteinDistanceParam{"foo", "oo", 1},
LevenshteinDistanceParam{"foo", "fao", 1},
LevenshteinDistanceParam{"foo", "abc", 3}
)
);
TEST(Suggestions, Trim) {
auto suggestions = Suggestions::bestMatches({"foooo", "bar", "fo", "gao"}, "foo");
auto onlyOne = suggestions.trim(1);
ASSERT_EQ(onlyOne.suggestions.size(), 1);
ASSERT_TRUE(onlyOne.suggestions.begin()->suggestion == "fo");
auto closest = suggestions.trim(999, 2);
ASSERT_EQ(closest.suggestions.size(), 3);
}
}

View file

@ -706,9 +706,14 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
auto [cursor, attrPath] = installable.getCursor(*evalState); auto [cursor, attrPath] = installable.getCursor(*evalState);
auto templateDir = cursor->getAttr("path")->getString(); auto templateDirAttr = cursor->getAttr("path");
auto templateDir = templateDirAttr->getString();
assert(store->isInStore(templateDir)); if (!store->isInStore(templateDir))
throw TypeError(
"'%s' was not found in the Nix store\n"
"If you've set '%s' to a string, try using a path instead.",
templateDir, templateDirAttr->getAttrPathStr());
std::vector<Path> files; std::vector<Path> files;

View file

@ -92,8 +92,9 @@ nix_tests = \
bash-profile.sh \ bash-profile.sh \
pass-as-file.sh \ pass-as-file.sh \
describe-stores.sh \ describe-stores.sh \
store-ping.sh \ nix-profile.sh \
nix-profile.sh suggestions.sh \
store-ping.sh
ifeq ($(HAVE_LIBCPUID), 1) ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh nix_tests += compute-levels.sh

36
tests/suggestions.sh Normal file
View file

@ -0,0 +1,36 @@
source common.sh
clearStore
cd "$TEST_HOME"
cat <<EOF > flake.nix
{
outputs = a: {
packages.$system = {
foo = 1;
fo1 = 1;
fo2 = 1;
fooo = 1;
foooo = 1;
fooooo = 1;
fooooo1 = 1;
fooooo2 = 1;
fooooo3 = 1;
fooooo4 = 1;
fooooo5 = 1;
fooooo6 = 1;
};
};
}
EOF
# Probable typo in the requested attribute path. Suggest some close possibilities
NIX_BUILD_STDERR_WITH_SUGGESTIONS=$(! nix build .\#fob 2>&1 1>/dev/null)
[[ "$NIX_BUILD_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \
fail "The nix build stderr should suggest the three closest possiblities"
# None of the possible attributes is close to `bar`, so shouldnt suggest anything
NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null)
[[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \
fail "The nix build stderr shouldnt suggest anything if theres nothing relevant to suggest"