nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-05 00:21:01 +01:00
Code Activity

Merge pull request #14540 from lovesegfault/pre-compute-outputgraph

Browse source 
perf(libstore/derivation-builder): pre-compute outputGraph for linear complexity
This commit is contained in:
John Ericson 2025-11-29 21:46:21 +00:00 • committed by GitHub
commit 01dbbc926f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 57 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/libstore/local-store.cc
View file

@ -989,10 +989,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
error if a cycle is detected and roll back the error if a cycle is detected and roll back the
transaction. Cycles can only occur when a derivation transaction. Cycles can only occur when a derivation
has multiple outputs. */ has multiple outputs. */
auto topoSortResult = topoSort(paths, {[&](const StorePath & path) { auto topoSortResult = topoSort(paths, [&](const StorePath & path) {
auto i = infos.find(path); auto i = infos.find(path);
return i == infos.end() ? StorePathSet() : i->second.references; return i == infos.end() ? StorePathSet() : i->second.references;
}}); });
std::visit( std::visit(
overloaded{ overloaded{

14
src/libstore/misc.cc
View file

@ -313,13 +313,13 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
StorePaths Store::topoSortPaths(const StorePathSet & paths) StorePaths Store::topoSortPaths(const StorePathSet & paths)
{ {
auto result = topoSort(paths, {[&](const StorePath & path) { auto result = topoSort(paths, [&](const StorePath & path) {
try { try {
return queryPathInfo(path)->references; return queryPathInfo(path)->references;
} catch (InvalidPath &) { } catch (InvalidPath &) {
return StorePathSet(); return StorePathSet();
} }
}}); });
return std::visit( return std::visit(
overloaded{ overloaded{

68
src/libstore/unix/build/derivation-builder.cc
View file

@ -1396,8 +1396,18 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
struct PerhapsNeedToRegister struct PerhapsNeedToRegister
{ {
StorePathSet refs; StorePathSet refs;
/**
* References to other outputs. Built by looking up in
* `scratchOutputsInverse`.
*/
StringSet otherOutputs;
}; };
/* inverse map of scratchOutputs for efficient lookup */
std::map<StorePath, std::string> scratchOutputsInverse;
for (auto & [outputName, path] : scratchOutputs)
scratchOutputsInverse.insert_or_assign(path, outputName);
std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered; std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
std::map<std::string, struct stat> outputStats; std::map<std::string, struct stat> outputStats;
for (auto & [outputName, _] : drv.outputs) { for (auto & [outputName, _] : drv.outputs) {
@ -1466,36 +1476,40 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
references = scanForReferences(blank, actualPath, referenceablePaths); references = scanForReferences(blank, actualPath, referenceablePaths);
} }
outputReferencesIfUnregistered.insert_or_assign(outputName, PerhapsNeedToRegister{.refs = references}); StringSet referencedOutputs;
for (auto & r : references)
if (auto * o = get(scratchOutputsInverse, r))
referencedOutputs.insert(*o);
outputReferencesIfUnregistered.insert_or_assign(
outputName,
PerhapsNeedToRegister{
.refs = references,
.otherOutputs = referencedOutputs,
});
outputStats.insert_or_assign(outputName, std::move(st)); outputStats.insert_or_assign(outputName, std::move(st));
} }
auto topoSortResult = topoSort(outputsToSort, {[&](const std::string & name) { StringSet emptySet;
auto orifu = get(outputReferencesIfUnregistered, name);
if (!orifu) auto topoSortResult = topoSort(outputsToSort, [&](const std::string & name) -> const StringSet & {
throw BuildError( auto * orifu = get(outputReferencesIfUnregistered, name);
BuildResult::Failure::OutputRejected, if (!orifu)
"no output reference for '%s' in build of '%s'", throw BuildError(
name, BuildResult::Failure::OutputRejected,
store.printStorePath(drvPath)); "no output reference for '%s' in build of '%s'",
return std::visit( name,
overloaded{ store.printStorePath(drvPath));
/* Since we'll use the already installed versions of these, we return std::visit(
can treat them as leaves and ignore any references they overloaded{
have. */ /* Since we'll use the already installed versions of these, we
[&](const AlreadyRegistered &) { return StringSet{}; }, can treat them as leaves and ignore any references they
[&](const PerhapsNeedToRegister & refs) { have. */
StringSet referencedOutputs; [&](const AlreadyRegistered &) -> const StringSet & { return emptySet; },
/* FIXME build inverted map up front so no quadratic waste here */ [&](const PerhapsNeedToRegister & refs) -> const StringSet & { return refs.otherOutputs; },
for (auto & r : refs.refs) },
for (auto & [o, p] : scratchOutputs) *orifu);
if (r == p) });
referencedOutputs.insert(o);
return referencedOutputs;
},
},
*orifu);
}});
auto sortedOutputNames = std::visit( auto sortedOutputNames = std::visit(
overloaded{ overloaded{

8
src/libutil/include/nix/util/topo-sort.hh
View file

@ -3,6 +3,7 @@
#include "nix/util/error.hh" #include "nix/util/error.hh"
#include <variant> #include <variant>
#include <concepts>
namespace nix { namespace nix {
@ -16,8 +17,9 @@ struct Cycle
template<typename T> template<typename T>
using TopoSortResult = std::variant<std::vector<T>, Cycle<T>>; using TopoSortResult = std::variant<std::vector<T>, Cycle<T>>;
template<typename T, typename Compare> template<typename T, typename Compare, std::invocable<const T &> F>
TopoSortResult<T> topoSort(std::set<T, Compare> items, std::function<std::set<T, Compare>(const T &)> getChildren) requires std::same_as<std::remove_cvref_t<std::invoke_result_t<F, const T &>>, std::set<T, Compare>>
TopoSortResult<T> topoSort(std::set<T, Compare> items, F && getChildren)
{ {
std::vector<T> sorted; std::vector<T> sorted;
decltype(items) visited, parents; decltype(items) visited, parents;
@ -34,7 +36,7 @@ TopoSortResult<T> topoSort(std::set<T, Compare> items, std::function<std::set<T,
} }
parents.insert(path); parents.insert(path);
auto references = getChildren(path); auto && references = std::invoke(getChildren, path);
for (auto & i : references) for (auto & i : references)
/* Don't traverse into items that don't exist in our starting set. */ /* Don't traverse into items that don't exist in our starting set. */