mirror of
https://github.com/NixOS/nix.git
synced 2025-11-11 04:56:01 +01:00
Merge branch 'master' (pre-reformat)
This commit is contained in:
Eelco Dolstra
commit
d23f9674bb
101 changed files with 1178 additions and 744 deletions
|
|
@ -1602,7 +1602,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
|
|||
symbols[i.name])
|
||||
.atPos(lambda.pos)
|
||||
.withTrace(pos, "from call site")
|
||||
.withFrame(*fun.lambda().env, lambda)
|
||||
.withFrame(*vCur.lambda().env, lambda)
|
||||
.debugThrow();
|
||||
}
|
||||
env2.values[displ++] = i.def->maybeThunk(*this, env2);
|
||||
|
|
@ -1629,7 +1629,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
|
|||
.atPos(lambda.pos)
|
||||
.withTrace(pos, "from call site")
|
||||
.withSuggestions(suggestions)
|
||||
.withFrame(*fun.lambda().env, lambda)
|
||||
.withFrame(*vCur.lambda().env, lambda)
|
||||
.debugThrow();
|
||||
}
|
||||
unreachable();
|
||||
|
|
|
|||
|
|
@ -211,6 +211,8 @@ struct EvalSettings : Config
|
|||
* `flamegraph` stack sampling profiler. Outputs folded format, one line per stack (suitable for `flamegraph.pl` and compatible tools).
|
||||
|
||||
Use [`eval-profile-file`](#conf-eval-profile-file) to specify where the profile is saved.
|
||||
|
||||
See [Using the `eval-profiler`](@docroot@/advanced-topics/eval-profiler.md).
|
||||
)"};
|
||||
|
||||
Setting<Path> evalProfileFile{this, "nix.profile", "eval-profile-file",
|
||||
|
|
|
|||
|
|
@ -7,12 +7,7 @@
|
|||
#include "nix/util/error.hh"
|
||||
|
||||
#include <boost/version.hpp>
|
||||
#define USE_FLAT_SYMBOL_SET (BOOST_VERSION >= 108100)
|
||||
#if USE_FLAT_SYMBOL_SET
|
||||
# include <boost/unordered/unordered_flat_set.hpp>
|
||||
#else
|
||||
# include <boost/unordered/unordered_set.hpp>
|
||||
#endif
|
||||
#include <boost/unordered/unordered_flat_set.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -214,12 +209,7 @@ private:
|
|||
* Transparent lookup of string view for a pointer to a ChunkedVector entry -> return offset into the store.
|
||||
* ChunkedVector references are never invalidated.
|
||||
*/
|
||||
#if USE_FLAT_SYMBOL_SET
|
||||
boost::unordered_flat_set<SymbolStr, SymbolStr::Hash, SymbolStr::Equal> symbols{SymbolStr::chunkSize};
|
||||
#else
|
||||
using SymbolValueAlloc = std::pmr::polymorphic_allocator<SymbolStr>;
|
||||
boost::unordered_set<SymbolStr, SymbolStr::Hash, SymbolStr::Equal, SymbolValueAlloc> symbols{SymbolStr::chunkSize, {&buffer}};
|
||||
#endif
|
||||
|
||||
public:
|
||||
|
||||
|
|
@ -230,19 +220,7 @@ public:
|
|||
// Most symbols are looked up more than once, so we trade off insertion performance
|
||||
// for lookup performance.
|
||||
// FIXME: make this thread-safe.
|
||||
return [&]<typename T>(T && key) -> Symbol {
|
||||
if constexpr (requires { symbols.insert<T>(key); }) {
|
||||
auto [it, _] = symbols.insert<T>(key);
|
||||
return Symbol(*it);
|
||||
} else {
|
||||
auto it = symbols.find<T>(key);
|
||||
if (it != symbols.end())
|
||||
return Symbol(*it);
|
||||
|
||||
it = symbols.emplace(key).first;
|
||||
return Symbol(*it);
|
||||
}
|
||||
}(SymbolStr::Key{store, s, stringAlloc});
|
||||
return Symbol(*symbols.insert(SymbolStr::Key{store, s, stringAlloc}).first);
|
||||
}
|
||||
|
||||
std::vector<SymbolStr> resolve(const std::vector<Symbol> & symbols) const
|
||||
|
|
@ -287,5 +265,3 @@ struct std::hash<nix::Symbol>
|
|||
return std::hash<decltype(s.id)>{}(s.id);
|
||||
}
|
||||
};
|
||||
|
||||
#undef USE_FLAT_SYMBOL_SET
|
||||
|
|
|
|||
|
|
@ -4122,9 +4122,9 @@ static RegisterPrimOp primop_lessThan({
|
|||
.name = "__lessThan",
|
||||
.args = {"e1", "e2"},
|
||||
.doc = R"(
|
||||
Return `true` if the number *e1* is less than the number *e2*, and
|
||||
`false` otherwise. Evaluation aborts if either *e1* or *e2* does not
|
||||
evaluate to a number.
|
||||
Return `true` if the value *e1* is less than the value *e2*, and `false` otherwise.
|
||||
Evaluation aborts if either *e1* or *e2* does not evaluate to a number, string or path.
|
||||
Furthermore, it aborts if *e2* does not match *e1*'s type according to the aforementioned classification of number, string or path.
|
||||
)",
|
||||
.fun = prim_lessThan,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
|
|||
for (auto & attr : *args[0]->attrs()) {
|
||||
const auto & attrName = state.symbols[attr.name];
|
||||
auto attrHint = [&]() -> std::string {
|
||||
return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure";
|
||||
return fmt("while evaluating the attribute '%s' passed to builtins.fetchClosure", attrName);
|
||||
};
|
||||
|
||||
if (attrName == "fromPath") {
|
||||
|
|
|
|||
|
|
@ -545,15 +545,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
|||
append(gitArgs, {"--depth", "1"});
|
||||
append(gitArgs, {std::string("--"), url, refspec});
|
||||
|
||||
runProgram(RunOptions {
|
||||
auto [status, output] = runProgram(RunOptions {
|
||||
.program = "git",
|
||||
.lookupPath = true,
|
||||
// FIXME: git stderr messes up our progress indicator, so
|
||||
// we're using --quiet for now. Should process its stderr.
|
||||
.args = gitArgs,
|
||||
.input = {},
|
||||
.mergeStderrToStdout = true,
|
||||
.isInteractive = true
|
||||
});
|
||||
|
||||
if (status > 0) {
|
||||
throw Error("Failed to fetch git repository %s : %s", url, output);
|
||||
}
|
||||
}
|
||||
|
||||
void verifyCommit(
|
||||
|
|
|
|||
|
|
@ -444,7 +444,11 @@ struct GitInputScheme : InputScheme
|
|||
// repo, treat as a remote URI to force a clone.
|
||||
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
|
||||
|
||||
// Why are we checking for bare repository?
|
||||
// well if it's a bare repository we want to force a git fetch rather than copying the folder
|
||||
bool isBareRepository = url.scheme == "file" && pathExists(url.path) &&
|
||||
!pathExists(url.path + "/.git");
|
||||
//
|
||||
// FIXME: here we turn a possibly relative path into an absolute path.
|
||||
// This allows relative git flake inputs to be resolved against the
|
||||
|
|
@ -462,6 +466,12 @@ struct GitInputScheme : InputScheme
|
|||
"See https://github.com/NixOS/nix/issues/12281 for details.",
|
||||
url);
|
||||
}
|
||||
|
||||
// If we don't check here for the path existence, then we can give libgit2 any directory
|
||||
// and it will initialize them as git directories.
|
||||
if (!pathExists(url.path)) {
|
||||
throw Error("The path '%s' does not exist.", url.path);
|
||||
}
|
||||
repoInfo.location = std::filesystem::absolute(url.path);
|
||||
} else {
|
||||
if (url.scheme == "file")
|
||||
|
|
@ -599,7 +609,7 @@ struct GitInputScheme : InputScheme
|
|||
? cacheDir / ref
|
||||
: cacheDir / "refs/heads" / ref;
|
||||
|
||||
bool doFetch;
|
||||
bool doFetch = false;
|
||||
time_t now = time(0);
|
||||
|
||||
/* If a rev was specified, we need to fetch if it's not in the
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#include "fetchers.hh"
|
||||
#include "nix/fetchers/fetchers.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
|
|
|
|||
|
|
@ -111,6 +111,25 @@ static DownloadTarballResult downloadTarball_(
|
|||
const Headers & headers,
|
||||
const std::string & displayPrefix)
|
||||
{
|
||||
|
||||
// Some friendly error messages for common mistakes.
|
||||
// Namely lets catch when the url is a local file path, but
|
||||
// it is not in fact a tarball.
|
||||
if (url.rfind("file://", 0) == 0) {
|
||||
// Remove "file://" prefix to get the local file path
|
||||
std::string localPath = url.substr(7);
|
||||
if (!std::filesystem::exists(localPath)) {
|
||||
throw Error("tarball '%s' does not exist.", localPath);
|
||||
}
|
||||
if (std::filesystem::is_directory(localPath)) {
|
||||
if (std::filesystem::exists(localPath + "/.git")) {
|
||||
throw Error(
|
||||
"tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath);
|
||||
}
|
||||
throw Error("tarball '%s' is a directory, not a file.", localPath);
|
||||
}
|
||||
}
|
||||
|
||||
Cache::Key cacheKey{"tarball", {{"url", url}}};
|
||||
|
||||
auto cached = settings.getCache()->lookupExpired(cacheKey);
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ test(
|
|||
env : {
|
||||
'_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data',
|
||||
'NIX_CONFIG': 'extra-experimental-features = flakes',
|
||||
'HOME': meson.current_build_dir() / 'test-home',
|
||||
},
|
||||
protocol : 'gtest',
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
buildPackages,
|
||||
stdenv,
|
||||
mkMesonExecutable,
|
||||
writableTmpDirAsHomeHook,
|
||||
|
||||
nix-flake,
|
||||
nix-flake-c,
|
||||
|
|
@ -55,19 +56,14 @@ mkMesonExecutable (finalAttrs: {
|
|||
runCommand "${finalAttrs.pname}-run"
|
||||
{
|
||||
meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
|
||||
buildInputs = [ writableTmpDirAsHomeHook ];
|
||||
}
|
||||
(
|
||||
lib.optionalString stdenv.hostPlatform.isWindows ''
|
||||
export HOME="$PWD/home-dir"
|
||||
mkdir -p "$HOME"
|
||||
''
|
||||
+ ''
|
||||
export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
|
||||
export NIX_CONFIG="extra-experimental-features = flakes"
|
||||
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
|
||||
touch $out
|
||||
''
|
||||
);
|
||||
(''
|
||||
export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
|
||||
export NIX_CONFIG="extra-experimental-features = flakes"
|
||||
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
|
||||
touch $out
|
||||
'');
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -715,16 +715,12 @@ LockedFlake lockFlake(
|
|||
Finally cleanup([&]() { parents.pop_back(); });
|
||||
|
||||
/* Recursively process the inputs of this
|
||||
flake. Also, unless we already have this flake
|
||||
in the top-level lock file, use this flake's
|
||||
own lock file. */
|
||||
flake, using its own lock file. */
|
||||
nodePaths.emplace(childNode, inputFlake.path.parent());
|
||||
computeLocks(
|
||||
inputFlake.inputs, childNode, inputAttrPath,
|
||||
oldLock
|
||||
? std::dynamic_pointer_cast<const Node>(oldLock)
|
||||
: readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(),
|
||||
oldLock ? followsPrefix : inputAttrPath,
|
||||
readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(),
|
||||
inputAttrPath,
|
||||
inputFlake.path,
|
||||
false);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,15 +35,17 @@ void printVersion(const std::string & programName);
|
|||
void printGCWarning();
|
||||
|
||||
class Store;
|
||||
struct MissingPaths;
|
||||
|
||||
void printMissing(
|
||||
ref<Store> store,
|
||||
const std::vector<DerivedPath> & paths,
|
||||
Verbosity lvl = lvlInfo);
|
||||
|
||||
void printMissing(ref<Store> store, const StorePathSet & willBuild,
|
||||
const StorePathSet & willSubstitute, const StorePathSet & unknown,
|
||||
uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
|
||||
void printMissing(
|
||||
ref<Store> store,
|
||||
const MissingPaths & missing,
|
||||
Verbosity lvl = lvlInfo);
|
||||
|
||||
std::string getArg(const std::string & opt,
|
||||
Strings::iterator & i, const Strings::iterator & end);
|
||||
|
|
|
|||
|
|
@ -46,43 +46,41 @@ void printGCWarning()
|
|||
|
||||
void printMissing(ref<Store> store, const std::vector<DerivedPath> & paths, Verbosity lvl)
|
||||
{
|
||||
uint64_t downloadSize, narSize;
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
|
||||
printMissing(store, store->queryMissing(paths), lvl);
|
||||
}
|
||||
|
||||
|
||||
void printMissing(ref<Store> store, const StorePathSet & willBuild,
|
||||
const StorePathSet & willSubstitute, const StorePathSet & unknown,
|
||||
uint64_t downloadSize, uint64_t narSize, Verbosity lvl)
|
||||
void printMissing(
|
||||
ref<Store> store,
|
||||
const MissingPaths & missing,
|
||||
Verbosity lvl)
|
||||
{
|
||||
if (!willBuild.empty()) {
|
||||
if (willBuild.size() == 1)
|
||||
if (!missing.willBuild.empty()) {
|
||||
if (missing.willBuild.size() == 1)
|
||||
printMsg(lvl, "this derivation will be built:");
|
||||
else
|
||||
printMsg(lvl, "these %d derivations will be built:", willBuild.size());
|
||||
auto sorted = store->topoSortPaths(willBuild);
|
||||
printMsg(lvl, "these %d derivations will be built:", missing.willBuild.size());
|
||||
auto sorted = store->topoSortPaths(missing.willBuild);
|
||||
reverse(sorted.begin(), sorted.end());
|
||||
for (auto & i : sorted)
|
||||
printMsg(lvl, " %s", store->printStorePath(i));
|
||||
}
|
||||
|
||||
if (!willSubstitute.empty()) {
|
||||
const float downloadSizeMiB = downloadSize / (1024.f * 1024.f);
|
||||
const float narSizeMiB = narSize / (1024.f * 1024.f);
|
||||
if (willSubstitute.size() == 1) {
|
||||
if (!missing.willSubstitute.empty()) {
|
||||
const float downloadSizeMiB = missing.downloadSize / (1024.f * 1024.f);
|
||||
const float narSizeMiB = missing.narSize / (1024.f * 1024.f);
|
||||
if (missing.willSubstitute.size() == 1) {
|
||||
printMsg(lvl, "this path will be fetched (%.2f MiB download, %.2f MiB unpacked):",
|
||||
downloadSizeMiB,
|
||||
narSizeMiB);
|
||||
} else {
|
||||
printMsg(lvl, "these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
|
||||
willSubstitute.size(),
|
||||
missing.willSubstitute.size(),
|
||||
downloadSizeMiB,
|
||||
narSizeMiB);
|
||||
}
|
||||
std::vector<const StorePath *> willSubstituteSorted = {};
|
||||
std::for_each(willSubstitute.begin(), willSubstitute.end(),
|
||||
std::for_each(missing.willSubstitute.begin(), missing.willSubstitute.end(),
|
||||
[&](const StorePath &p) { willSubstituteSorted.push_back(&p); });
|
||||
std::sort(willSubstituteSorted.begin(), willSubstituteSorted.end(),
|
||||
[](const StorePath *lhs, const StorePath *rhs) {
|
||||
|
|
@ -95,10 +93,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
|
|||
printMsg(lvl, " %s", store->printStorePath(*p));
|
||||
}
|
||||
|
||||
if (!unknown.empty()) {
|
||||
if (!missing.unknown.empty()) {
|
||||
printMsg(lvl, "don't know how to build these paths%s:",
|
||||
(settings.readOnlyMode ? " (may be caused by read-only store access)" : ""));
|
||||
for (auto & i : unknown)
|
||||
for (auto & i : missing.unknown)
|
||||
printMsg(lvl, " %s", store->printStorePath(i));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,6 +100,8 @@ test(
|
|||
this_exe,
|
||||
env : {
|
||||
'_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data',
|
||||
'HOME': meson.current_build_dir() / 'test-home',
|
||||
'NIX_REMOTE': meson.current_build_dir() / 'test-home' / 'store',
|
||||
},
|
||||
protocol : 'gtest',
|
||||
)
|
||||
|
|
|
|||
|
|
@ -28,10 +28,6 @@ TEST_F(nix_api_store_test, nix_store_get_uri)
|
|||
|
||||
TEST_F(nix_api_util_context, nix_store_get_storedir_default)
|
||||
{
|
||||
if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") {
|
||||
// skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles
|
||||
GTEST_SKIP();
|
||||
}
|
||||
nix_libstore_init(ctx);
|
||||
Store * store = nix_store_open(ctx, nullptr, nullptr);
|
||||
assert_ctx_ok();
|
||||
|
|
@ -141,10 +137,6 @@ TEST_F(nix_api_store_test, nix_store_real_path)
|
|||
|
||||
TEST_F(nix_api_util_context, nix_store_real_path_relocated)
|
||||
{
|
||||
if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") {
|
||||
// Can't open default store from within sandbox
|
||||
GTEST_SKIP();
|
||||
}
|
||||
auto tmp = nix::createTempDir();
|
||||
std::string storeRoot = tmp + "/store";
|
||||
std::string stateDir = tmp + "/state";
|
||||
|
|
@ -184,13 +176,7 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated)
|
|||
|
||||
TEST_F(nix_api_util_context, nix_store_real_path_binary_cache)
|
||||
{
|
||||
if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") {
|
||||
// TODO: override NIX_CACHE_HOME?
|
||||
// skipping test in sandbox because narinfo cache can't be written
|
||||
GTEST_SKIP();
|
||||
}
|
||||
|
||||
Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr);
|
||||
Store * store = nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir()).c_str(), nullptr);
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(store, nullptr);
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
buildPackages,
|
||||
stdenv,
|
||||
mkMesonExecutable,
|
||||
writableTmpDirAsHomeHook,
|
||||
|
||||
nix-store,
|
||||
nix-store-c,
|
||||
|
|
@ -72,18 +73,14 @@ mkMesonExecutable (finalAttrs: {
|
|||
runCommand "${finalAttrs.pname}-run"
|
||||
{
|
||||
meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
|
||||
buildInputs = [ writableTmpDirAsHomeHook ];
|
||||
}
|
||||
(
|
||||
lib.optionalString stdenv.hostPlatform.isWindows ''
|
||||
export HOME="$PWD/home-dir"
|
||||
mkdir -p "$HOME"
|
||||
''
|
||||
+ ''
|
||||
export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"}
|
||||
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
|
||||
touch $out
|
||||
''
|
||||
);
|
||||
(''
|
||||
export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"}
|
||||
export NIX_REMOTE=$HOME/store
|
||||
${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
|
||||
touch $out
|
||||
'');
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#include "nix/store/build/derivation-building-goal.hh"
|
||||
#include "nix/store/build/derivation-goal.hh"
|
||||
#include "nix/store/build/derivation-trampoline-goal.hh"
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
# include "nix/store/build/hook-instance.hh"
|
||||
# include "nix/store/build/derivation-builder.hh"
|
||||
|
|
@ -72,7 +72,7 @@ std::string DerivationBuildingGoal::key()
|
|||
/* Ensure that derivations get built in order of their name,
|
||||
i.e. a derivation named "aardvark" always comes before
|
||||
"baboon". And substitution goals always happen before
|
||||
derivation goals (due to "b$"). */
|
||||
derivation goals (due to "bd$"). */
|
||||
return "bd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
|
||||
}
|
||||
|
||||
|
|
@ -266,7 +266,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
|
|||
auto mEntry = get(inputGoals, drvPath);
|
||||
if (!mEntry) return std::nullopt;
|
||||
|
||||
auto buildResult = (*mEntry)->getBuildResult(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}});
|
||||
auto & buildResult = (*mEntry)->buildResult;
|
||||
if (!buildResult.success()) return std::nullopt;
|
||||
|
||||
auto i = get(buildResult.builtOutputs, outputName);
|
||||
|
|
@ -296,9 +296,11 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
|
|||
worker.store.printStorePath(pathResolved),
|
||||
});
|
||||
|
||||
// FIXME wanted outputs
|
||||
auto resolvedDrvGoal = worker.makeDerivationGoal(
|
||||
makeConstantStorePathRef(pathResolved), OutputsSpec::All{}, buildMode);
|
||||
/* TODO https://github.com/NixOS/nix/issues/13247 we should
|
||||
let the calling goal do this, so it has a change to pass
|
||||
just the output(s) it cares about. */
|
||||
auto resolvedDrvGoal = worker.makeDerivationTrampolineGoal(
|
||||
pathResolved, OutputsSpec::All{}, drvResolved, buildMode);
|
||||
{
|
||||
Goals waitees{resolvedDrvGoal};
|
||||
co_await await(std::move(waitees));
|
||||
|
|
@ -306,20 +308,16 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
|
|||
|
||||
trace("resolved derivation finished");
|
||||
|
||||
auto resolvedDrv = *resolvedDrvGoal->drv;
|
||||
auto resolvedResult = resolvedDrvGoal->getBuildResult(DerivedPath::Built{
|
||||
.drvPath = makeConstantStorePathRef(pathResolved),
|
||||
.outputs = OutputsSpec::All{},
|
||||
});
|
||||
auto resolvedResult = resolvedDrvGoal->buildResult;
|
||||
|
||||
SingleDrvOutputs builtOutputs;
|
||||
|
||||
if (resolvedResult.success()) {
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, drvResolved);
|
||||
|
||||
StorePathSet outputPaths;
|
||||
|
||||
for (auto & outputName : resolvedDrv.outputNames()) {
|
||||
for (auto & outputName : drvResolved.outputNames()) {
|
||||
auto initialOutput = get(initialOutputs, outputName);
|
||||
auto resolvedHash = get(resolvedHashes, outputName);
|
||||
if ((!initialOutput) || (!resolvedHash))
|
||||
|
|
@ -340,7 +338,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
|
|||
|
||||
throw Error(
|
||||
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/realisation)",
|
||||
resolvedDrvGoal->drvReq->to_string(worker.store), outputName);
|
||||
worker.store.printStorePath(pathResolved), outputName);
|
||||
}();
|
||||
|
||||
if (!drv->type().isImpure()) {
|
||||
|
|
|
|||
|
|
@ -24,35 +24,18 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
DerivationGoal::DerivationGoal(ref<const SingleDerivedPath> drvReq,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker, loadDerivation())
|
||||
, drvReq(drvReq)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
name = fmt(
|
||||
"building of '%s' from .drv file",
|
||||
DerivedPath::Built { drvReq, wantedOutputs }.to_string(worker.store));
|
||||
trace("created");
|
||||
|
||||
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
|
||||
worker.updateProgress();
|
||||
}
|
||||
|
||||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker, haveDerivation(drvPath))
|
||||
, drvReq(makeConstantStorePathRef(drvPath))
|
||||
, wantedOutputs(wantedOutputs)
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath, const Derivation & drv,
|
||||
const OutputName & wantedOutput, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker, haveDerivation())
|
||||
, drvPath(drvPath)
|
||||
, wantedOutput(wantedOutput)
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
this->drv = std::make_unique<Derivation>(drv);
|
||||
|
||||
name = fmt(
|
||||
"building of '%s' from in-memory derivation",
|
||||
DerivedPath::Built { drvReq, drv.outputNames() }.to_string(worker.store));
|
||||
DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
|
||||
trace("created");
|
||||
|
||||
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
|
||||
|
|
@ -61,114 +44,20 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
|
|||
}
|
||||
|
||||
|
||||
static StorePath pathPartOfReq(const SingleDerivedPath & req)
|
||||
{
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[&](const SingleDerivedPath::Opaque & bo) { return bo.path; },
|
||||
[&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); },
|
||||
},
|
||||
req.raw());
|
||||
}
|
||||
|
||||
|
||||
std::string DerivationGoal::key()
|
||||
{
|
||||
/* Ensure that derivations get built in order of their name,
|
||||
i.e. a derivation named "aardvark" always comes before
|
||||
"baboon". And substitution goals always happen before
|
||||
derivation goals (due to "b$"). */
|
||||
return "b$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
|
||||
return "b$" + std::string(drvPath.name()) + "$" + SingleDerivedPath::Built{
|
||||
.drvPath = makeConstantStorePathRef(drvPath),
|
||||
.output = wantedOutput,
|
||||
}.to_string(worker.store);
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
|
||||
{
|
||||
auto newWanted = wantedOutputs.union_(outputs);
|
||||
switch (needRestart) {
|
||||
case NeedRestartForMoreOutputs::OutputsUnmodifiedDontNeed:
|
||||
if (!newWanted.isSubsetOf(wantedOutputs))
|
||||
needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed;
|
||||
break;
|
||||
case NeedRestartForMoreOutputs::OutputsAddedDoNeed:
|
||||
/* No need to check whether we added more outputs, because a
|
||||
restart is already queued up. */
|
||||
break;
|
||||
case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed:
|
||||
/* We are already building all outputs, so it doesn't matter if
|
||||
we now want more. */
|
||||
break;
|
||||
};
|
||||
wantedOutputs = newWanted;
|
||||
}
|
||||
|
||||
|
||||
Goal::Co DerivationGoal::loadDerivation() {
|
||||
trace("need to load derivation from file");
|
||||
|
||||
{
|
||||
/* The first thing to do is to make sure that the derivation
|
||||
exists. If it doesn't, it may be built from another
|
||||
derivation, or merely substituted. We can make goal to get it
|
||||
and not worry about which method it takes to get the
|
||||
derivation. */
|
||||
|
||||
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
|
||||
if (buildMode != bmNormal)
|
||||
return std::nullopt;
|
||||
|
||||
auto drvPath = StorePath::dummy;
|
||||
try {
|
||||
drvPath = resolveDerivedPath(worker.store, *drvReq);
|
||||
} catch (MissingRealisation &) {
|
||||
return std::nullopt;
|
||||
}
|
||||
auto cond = worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath);
|
||||
return cond ? std::optional{drvPath} : std::nullopt;
|
||||
}()) {
|
||||
trace(
|
||||
fmt("already have drv '%s' for '%s', can go straight to building",
|
||||
worker.store.printStorePath(*optDrvPath),
|
||||
drvReq->to_string(worker.store)));
|
||||
} else {
|
||||
trace("need to obtain drv we want to build");
|
||||
Goals waitees{worker.makeGoal(DerivedPath::fromSingle(*drvReq))};
|
||||
co_await await(std::move(waitees));
|
||||
}
|
||||
|
||||
trace("loading derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
|
||||
}
|
||||
|
||||
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
|
||||
|
||||
/* `drvPath' should already be a root, but let's be on the safe
|
||||
side: if the user forgot to make it a root, we wouldn't want
|
||||
things being garbage collected while we're busy. */
|
||||
worker.evalStore.addTempRoot(drvPath);
|
||||
|
||||
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
|
||||
|
||||
- Resolved derivation are resolved against main store realisations, and so must be stored there.
|
||||
|
||||
- Dynamic derivations are built, and so are found in the main store.
|
||||
*/
|
||||
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
|
||||
if (drvStore->isValidPath(drvPath)) {
|
||||
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert(drv);
|
||||
|
||||
co_return haveDerivation(drvPath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
|
||||
Goal::Co DerivationGoal::haveDerivation()
|
||||
{
|
||||
trace("have derivation");
|
||||
|
||||
|
|
@ -205,18 +94,26 @@ Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
|
|||
|
||||
trace("outer build done");
|
||||
|
||||
buildResult = g->getBuildResult(DerivedPath::Built{
|
||||
.drvPath = makeConstantStorePathRef(drvPath),
|
||||
.outputs = wantedOutputs,
|
||||
});
|
||||
buildResult = g->buildResult;
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
/* In checking mode, the builder will not register any outputs.
|
||||
So we want to make sure the ones that we wanted to check are
|
||||
properly there. */
|
||||
buildResult.builtOutputs = assertPathValidity(drvPath);
|
||||
buildResult.builtOutputs = assertPathValidity();
|
||||
}
|
||||
|
||||
for (auto it = buildResult.builtOutputs.begin(); it != buildResult.builtOutputs.end(); ) {
|
||||
if (it->first != wantedOutput) {
|
||||
it = buildResult.builtOutputs.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
if (buildResult.success())
|
||||
assert(buildResult.builtOutputs.count(wantedOutput) > 0);
|
||||
|
||||
co_return amDone(g->exitCode, g->ex);
|
||||
};
|
||||
|
||||
|
|
@ -261,11 +158,11 @@ Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
|
|||
|
||||
{
|
||||
/* Check what outputs paths are not already valid. */
|
||||
auto [allValid, validOutputs] = checkPathValidity(drvPath);
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
/* If they are all valid, then we're done. */
|
||||
if (allValid && buildMode == bmNormal) {
|
||||
co_return done(drvPath, BuildResult::AlreadyValid, std::move(validOutputs));
|
||||
co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -302,25 +199,20 @@ Goal::Co DerivationGoal::haveDerivation(StorePath drvPath)
|
|||
assert(!drv->type().isImpure());
|
||||
|
||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) {
|
||||
co_return done(drvPath, BuildResult::TransientFailure, {},
|
||||
co_return done(BuildResult::TransientFailure, {},
|
||||
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||
worker.store.printStorePath(drvPath)));
|
||||
}
|
||||
|
||||
nrFailed = nrNoSubstituters = 0;
|
||||
|
||||
if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) {
|
||||
needRestart = NeedRestartForMoreOutputs::OutputsUnmodifiedDontNeed;
|
||||
co_return haveDerivation(std::move(drvPath));
|
||||
}
|
||||
|
||||
auto [allValid, validOutputs] = checkPathValidity(drvPath);
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
|
||||
if (buildMode == bmNormal && allValid) {
|
||||
co_return done(drvPath, BuildResult::Substituted, std::move(validOutputs));
|
||||
co_return done(BuildResult::Substituted, std::move(validOutputs));
|
||||
}
|
||||
if (buildMode == bmRepair && allValid) {
|
||||
co_return repairClosure(std::move(drvPath));
|
||||
co_return repairClosure();
|
||||
}
|
||||
if (buildMode == bmCheck && !allValid)
|
||||
throw Error("some outputs of '%s' are not valid, so checking is not possible",
|
||||
|
|
@ -343,7 +235,7 @@ struct value_comparison
|
|||
};
|
||||
|
||||
|
||||
Goal::Co DerivationGoal::repairClosure(StorePath drvPath)
|
||||
Goal::Co DerivationGoal::repairClosure()
|
||||
{
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
|
|
@ -353,11 +245,10 @@ Goal::Co DerivationGoal::repairClosure(StorePath drvPath)
|
|||
that produced those outputs. */
|
||||
|
||||
/* Get the output closure. */
|
||||
auto outputs = queryDerivationOutputMap(drvPath);
|
||||
auto outputs = queryDerivationOutputMap();
|
||||
StorePathSet outputClosure;
|
||||
for (auto & i : outputs) {
|
||||
if (!wantedOutputs.contains(i.first)) continue;
|
||||
worker.store.computeFSClosure(i.second, outputClosure);
|
||||
if (auto mPath = get(outputs, wantedOutput)) {
|
||||
worker.store.computeFSClosure(*mPath, outputClosure);
|
||||
}
|
||||
|
||||
/* Filter out our own outputs (which we have already checked). */
|
||||
|
|
@ -411,11 +302,11 @@ Goal::Co DerivationGoal::repairClosure(StorePath drvPath)
|
|||
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
|
||||
worker.store.printStorePath(drvPath));
|
||||
}
|
||||
co_return done(drvPath, BuildResult::AlreadyValid, assertPathValidity(drvPath));
|
||||
co_return done(BuildResult::AlreadyValid, assertPathValidity());
|
||||
}
|
||||
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap(const StorePath & drvPath)
|
||||
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
|
||||
{
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
|
|
@ -431,7 +322,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
|
|||
return res;
|
||||
}
|
||||
|
||||
OutputPathMap DerivationGoal::queryDerivationOutputMap(const StorePath & drvPath)
|
||||
OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
||||
{
|
||||
assert(!drv->type().isImpure());
|
||||
|
||||
|
|
@ -447,28 +338,21 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap(const StorePath & drvPath
|
|||
}
|
||||
|
||||
|
||||
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity(const StorePath & drvPath)
|
||||
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
|
||||
{
|
||||
if (drv->type().isImpure()) return { false, {} };
|
||||
|
||||
bool checkHash = buildMode == bmRepair;
|
||||
auto wantedOutputsLeft = std::visit(overloaded {
|
||||
[&](const OutputsSpec::All &) {
|
||||
return StringSet {};
|
||||
},
|
||||
[&](const OutputsSpec::Names & names) {
|
||||
return static_cast<StringSet>(names);
|
||||
},
|
||||
}, wantedOutputs.raw);
|
||||
StringSet wantedOutputsLeft{wantedOutput};
|
||||
SingleDrvOutputs validOutputs;
|
||||
|
||||
for (auto & i : queryPartialDerivationOutputMap(drvPath)) {
|
||||
for (auto & i : queryPartialDerivationOutputMap()) {
|
||||
auto initialOutput = get(initialOutputs, i.first);
|
||||
if (!initialOutput)
|
||||
// this is an invalid output, gets caught with (!wantedOutputsLeft.empty())
|
||||
continue;
|
||||
auto & info = *initialOutput;
|
||||
info.wanted = wantedOutputs.contains(i.first);
|
||||
info.wanted = wantedOutput == i.first;
|
||||
if (info.wanted)
|
||||
wantedOutputsLeft.erase(i.first);
|
||||
if (i.second) {
|
||||
|
|
@ -527,9 +411,9 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity(const StoreP
|
|||
}
|
||||
|
||||
|
||||
SingleDrvOutputs DerivationGoal::assertPathValidity(const StorePath & drvPath)
|
||||
SingleDrvOutputs DerivationGoal::assertPathValidity()
|
||||
{
|
||||
auto [allValid, validOutputs] = checkPathValidity(drvPath);
|
||||
auto [allValid, validOutputs] = checkPathValidity();
|
||||
if (!allValid)
|
||||
throw Error("some outputs are unexpectedly invalid");
|
||||
return validOutputs;
|
||||
|
|
@ -537,7 +421,6 @@ SingleDrvOutputs DerivationGoal::assertPathValidity(const StorePath & drvPath)
|
|||
|
||||
|
||||
Goal::Done DerivationGoal::done(
|
||||
const StorePath & drvPath,
|
||||
BuildResult::Status status,
|
||||
SingleDrvOutputs builtOutputs,
|
||||
std::optional<Error> ex)
|
||||
|
|
@ -553,7 +436,7 @@ Goal::Done DerivationGoal::done(
|
|||
mcExpectedBuilds.reset();
|
||||
|
||||
if (buildResult.success()) {
|
||||
auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
|
||||
auto wantedBuiltOutputs = filterDrvOutputs(OutputsSpec::Names{wantedOutput}, std::move(builtOutputs));
|
||||
assert(!wantedBuiltOutputs.empty());
|
||||
buildResult.builtOutputs = std::move(wantedBuiltOutputs);
|
||||
if (status == BuildResult::Built)
|
||||
|
|
|
|||
175
src/libstore/build/derivation-trampoline-goal.cc
Normal file
175
src/libstore/build/derivation-trampoline-goal.cc
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
#include "nix/store/build/derivation-trampoline-goal.hh"
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/derivations.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
DerivationTrampolineGoal::DerivationTrampolineGoal(
|
||||
ref<const SingleDerivedPath> drvReq, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker, init())
|
||||
, drvReq(drvReq)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
commonInit();
|
||||
}
|
||||
|
||||
DerivationTrampolineGoal::DerivationTrampolineGoal(
|
||||
const StorePath & drvPath,
|
||||
const OutputsSpec & wantedOutputs,
|
||||
const Derivation & drv,
|
||||
Worker & worker,
|
||||
BuildMode buildMode)
|
||||
: Goal(worker, haveDerivation(drvPath, drv))
|
||||
, drvReq(makeConstantStorePathRef(drvPath))
|
||||
, wantedOutputs(wantedOutputs)
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
commonInit();
|
||||
}
|
||||
|
||||
void DerivationTrampolineGoal::commonInit()
|
||||
{
|
||||
name =
|
||||
fmt("outer obtaining drv from '%s' and then building outputs %s",
|
||||
drvReq->to_string(worker.store),
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const OutputsSpec::All) -> std::string { return "* (all of them)"; },
|
||||
[&](const OutputsSpec::Names os) { return concatStringsSep(", ", quoteStrings(os)); },
|
||||
},
|
||||
wantedOutputs.raw));
|
||||
trace("created outer");
|
||||
|
||||
worker.updateProgress();
|
||||
}
|
||||
|
||||
DerivationTrampolineGoal::~DerivationTrampolineGoal() {}
|
||||
|
||||
static StorePath pathPartOfReq(const SingleDerivedPath & req)
|
||||
{
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[&](const SingleDerivedPath::Opaque & bo) { return bo.path; },
|
||||
[&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); },
|
||||
},
|
||||
req.raw());
|
||||
}
|
||||
|
||||
std::string DerivationTrampolineGoal::key()
|
||||
{
|
||||
/* Ensure that derivations get built in order of their name,
|
||||
i.e. a derivation named "aardvark" always comes before "baboon". And
|
||||
substitution goals, derivation goals, and derivation building goals always happen before
|
||||
derivation goals (due to "bt$"). */
|
||||
return "bt$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + DerivedPath::Built{
|
||||
.drvPath = drvReq,
|
||||
.outputs = wantedOutputs,
|
||||
}.to_string(worker.store);
|
||||
}
|
||||
|
||||
void DerivationTrampolineGoal::timedOut(Error && ex) {}
|
||||
|
||||
Goal::Co DerivationTrampolineGoal::init()
|
||||
{
|
||||
trace("need to load derivation from file");
|
||||
|
||||
/* The first thing to do is to make sure that the derivation
|
||||
exists. If it doesn't, it may be built from another derivation,
|
||||
or merely substituted. We can make goal to get it and not worry
|
||||
about which method it takes to get the derivation. */
|
||||
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
|
||||
if (buildMode != bmNormal)
|
||||
return std::nullopt;
|
||||
|
||||
auto drvPath = StorePath::dummy;
|
||||
try {
|
||||
drvPath = resolveDerivedPath(worker.store, *drvReq);
|
||||
} catch (MissingRealisation &) {
|
||||
return std::nullopt;
|
||||
}
|
||||
auto cond = worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath);
|
||||
return cond ? std::optional{drvPath} : std::nullopt;
|
||||
}()) {
|
||||
trace(
|
||||
fmt("already have drv '%s' for '%s', can go straight to building",
|
||||
worker.store.printStorePath(*optDrvPath),
|
||||
drvReq->to_string(worker.store)));
|
||||
} else {
|
||||
trace("need to obtain drv we want to build");
|
||||
Goals waitees{worker.makeGoal(DerivedPath::fromSingle(*drvReq))};
|
||||
co_await await(std::move(waitees));
|
||||
}
|
||||
|
||||
trace("outer load and build derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
|
||||
}
|
||||
|
||||
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
|
||||
|
||||
/* `drvPath' should already be a root, but let's be on the safe
|
||||
side: if the user forgot to make it a root, we wouldn't want
|
||||
things being garbage collected while we're busy. */
|
||||
worker.evalStore.addTempRoot(drvPath);
|
||||
|
||||
/* Get the derivation. It is probably in the eval store, but it might be in the main store:
|
||||
|
||||
- Resolved derivation are resolved against main store realisations, and so must be stored there.
|
||||
|
||||
- Dynamic derivations are built, and so are found in the main store.
|
||||
*/
|
||||
auto drv = [&] {
|
||||
for (auto * drvStore : {&worker.evalStore, &worker.store})
|
||||
if (drvStore->isValidPath(drvPath))
|
||||
return drvStore->readDerivation(drvPath);
|
||||
assert(false);
|
||||
}();
|
||||
|
||||
co_return haveDerivation(std::move(drvPath), std::move(drv));
|
||||
}
|
||||
|
||||
Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation drv)
|
||||
{
|
||||
trace("have derivation, will kick off derivations goals per wanted output");
|
||||
|
||||
auto resolvedWantedOutputs = std::visit(
|
||||
overloaded{
|
||||
[&](const OutputsSpec::Names & names) -> OutputsSpec::Names { return names; },
|
||||
[&](const OutputsSpec::All &) -> OutputsSpec::Names {
|
||||
StringSet outputs;
|
||||
for (auto & [outputName, _] : drv.outputs)
|
||||
outputs.insert(outputName);
|
||||
return outputs;
|
||||
},
|
||||
},
|
||||
wantedOutputs.raw);
|
||||
|
||||
Goals concreteDrvGoals;
|
||||
|
||||
/* Build this step! */
|
||||
|
||||
for (auto & output : resolvedWantedOutputs) {
|
||||
auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode));
|
||||
g->preserveException = true;
|
||||
/* We will finish with it ourselves, as if we were the derivational goal. */
|
||||
concreteDrvGoals.insert(std::move(g));
|
||||
}
|
||||
|
||||
// Copy on purpose
|
||||
co_await await(Goals(concreteDrvGoals));
|
||||
|
||||
trace("outer build done");
|
||||
|
||||
auto & g = *concreteDrvGoals.begin();
|
||||
buildResult = g->buildResult;
|
||||
for (auto & g2 : concreteDrvGoals) {
|
||||
for (auto && [x, y] : g2->buildResult.builtOutputs)
|
||||
buildResult.builtOutputs.insert_or_assign(x, y);
|
||||
}
|
||||
|
||||
co_return amDone(g->exitCode, g->ex);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,8 +1,7 @@
|
|||
#include "nix/store/derivations.hh"
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/build/substitution-goal.hh"
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
# include "nix/store/build/derivation-goal.hh"
|
||||
#endif
|
||||
#include "nix/store/build/derivation-trampoline-goal.hh"
|
||||
#include "nix/store/local-store.hh"
|
||||
#include "nix/util/strings.hh"
|
||||
|
||||
|
|
@ -28,12 +27,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
|
|||
ex = std::move(i->ex);
|
||||
}
|
||||
if (i->exitCode != Goal::ecSuccess) {
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
|
||||
if (auto i2 = dynamic_cast<DerivationTrampolineGoal *>(i.get()))
|
||||
failed.insert(i2->drvReq->to_string(*this));
|
||||
else
|
||||
#endif
|
||||
if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
|
||||
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
|
||||
failed.insert(printStorePath(i2->storePath));
|
||||
}
|
||||
}
|
||||
|
|
@ -70,7 +66,7 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
|
|||
|
||||
for (auto & [req, goalPtr] : state)
|
||||
results.emplace_back(KeyedBuildResult {
|
||||
goalPtr->getBuildResult(req),
|
||||
goalPtr->buildResult,
|
||||
/* .path = */ req,
|
||||
});
|
||||
|
||||
|
|
@ -81,19 +77,11 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
|
|||
BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this, *this);
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
|
||||
#else
|
||||
std::shared_ptr<Goal> goal;
|
||||
throw UnimplementedError("Building derivations not yet implemented on windows.");
|
||||
#endif
|
||||
auto goal = worker.makeDerivationTrampolineGoal(drvPath, OutputsSpec::All {}, drv, buildMode);
|
||||
|
||||
try {
|
||||
worker.run(Goals{goal});
|
||||
return goal->getBuildResult(DerivedPath::Built {
|
||||
.drvPath = makeConstantStorePathRef(drvPath),
|
||||
.outputs = OutputsSpec::All {},
|
||||
});
|
||||
return goal->buildResult;
|
||||
} catch (Error & e) {
|
||||
return BuildResult {
|
||||
.status = BuildResult::MiscFailure,
|
||||
|
|
|
|||
|
|
@ -101,30 +101,6 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
|
|||
return s1 < s2;
|
||||
}
|
||||
|
||||
|
||||
BuildResult Goal::getBuildResult(const DerivedPath & req) const {
|
||||
BuildResult res { buildResult };
|
||||
|
||||
if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {
|
||||
auto & bp = *pbp;
|
||||
|
||||
/* Because goals are in general shared between derived paths
|
||||
that share the same derivation, we need to filter their
|
||||
results to get back just the results we care about.
|
||||
*/
|
||||
|
||||
for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) {
|
||||
if (bp.outputs.contains(it->first))
|
||||
++it;
|
||||
else
|
||||
it = res.builtOutputs.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
|
||||
{
|
||||
if (goals.find(p) != goals.end())
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||
#include "nix/store/build/derivation-goal.hh"
|
||||
#include "nix/store/build/derivation-building-goal.hh"
|
||||
#include "nix/store/build/derivation-trampoline-goal.hh"
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
# include "nix/store/build/hook-instance.hh"
|
||||
#endif
|
||||
|
|
@ -53,52 +54,40 @@ std::shared_ptr<G> Worker::initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args &
|
|||
return goal;
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
||||
std::shared_ptr<DerivationTrampolineGoal> Worker::makeDerivationTrampolineGoal(
|
||||
ref<const SingleDerivedPath> drvReq,
|
||||
const OutputsSpec & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
|
||||
BuildMode buildMode)
|
||||
{
|
||||
std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals.ensureSlot(*drvReq).value;
|
||||
std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
|
||||
if (!goal) {
|
||||
goal = mkDrvGoal();
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
} else {
|
||||
goal->addWantedOutputs(wantedOutputs);
|
||||
}
|
||||
return goal;
|
||||
return initGoalIfNeeded(
|
||||
derivationTrampolineGoals.ensureSlot(*drvReq).value[wantedOutputs],
|
||||
drvReq, wantedOutputs, *this, buildMode);
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(ref<const SingleDerivedPath> drvReq,
|
||||
const OutputsSpec & wantedOutputs, BuildMode buildMode)
|
||||
std::shared_ptr<DerivationTrampolineGoal> Worker::makeDerivationTrampolineGoal(
|
||||
const StorePath & drvPath,
|
||||
const OutputsSpec & wantedOutputs,
|
||||
const Derivation & drv,
|
||||
BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvReq, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
|
||||
return std::make_shared<DerivationGoal>(drvReq, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
return initGoalIfNeeded(
|
||||
derivationTrampolineGoals.ensureSlot(DerivedPath::Opaque{drvPath}).value[wantedOutputs],
|
||||
drvPath, wantedOutputs, drv, *this, buildMode);
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
|
||||
const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
|
||||
const Derivation & drv, const OutputName & wantedOutput, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(makeConstantStorePathRef(drvPath), wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
|
||||
return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
return initGoalIfNeeded(derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode);
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationBuildingGoal> Worker::makeDerivationBuildingGoal(const StorePath & drvPath,
|
||||
const Derivation & drv, BuildMode buildMode)
|
||||
{
|
||||
std::weak_ptr<DerivationBuildingGoal> & goal_weak = derivationBuildingGoals[drvPath];
|
||||
auto goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<DerivationBuildingGoal>(drvPath, drv, *this, buildMode);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, drv, *this, buildMode);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -118,7 +107,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
|
|||
{
|
||||
return std::visit(overloaded {
|
||||
[&](const DerivedPath::Built & bfd) -> GoalPtr {
|
||||
return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode);
|
||||
return makeDerivationTrampolineGoal(bfd.drvPath, bfd.outputs, buildMode);
|
||||
},
|
||||
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
|
||||
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
|
||||
|
|
@ -126,46 +115,52 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
|
|||
}, req.raw());
|
||||
}
|
||||
|
||||
|
||||
template<typename K, typename V, typename F>
|
||||
static void cullMap(std::map<K, V> & goalMap, F f)
|
||||
/**
|
||||
* This function is polymorphic (both via type parameters and
|
||||
* overloading) and recursive in order to work on a various types of
|
||||
* trees
|
||||
*
|
||||
* @return Whether the tree node we are processing is not empty / should
|
||||
* be kept alive. In the case of this overloading the node in question
|
||||
* is the leaf, the weak reference itself. If the weak reference points
|
||||
* to the goal we are looking for, our caller can delete it. In the
|
||||
* inductive case where the node is an interior node, we'll likewise
|
||||
* return whether the interior node is non-empty. If it is empty
|
||||
* (because we just deleted its last child), then our caller can
|
||||
* likewise delete it.
|
||||
*/
|
||||
template<typename G>
|
||||
static bool removeGoal(std::shared_ptr<G> goal, std::weak_ptr<G> & gp)
|
||||
{
|
||||
for (auto i = goalMap.begin(); i != goalMap.end();)
|
||||
if (!f(i->second))
|
||||
return gp.lock() != goal;
|
||||
}
|
||||
|
||||
template<typename K, typename G, typename Inner>
|
||||
static bool removeGoal(std::shared_ptr<G> goal, std::map<K, Inner> & goalMap)
|
||||
{
|
||||
/* !!! inefficient */
|
||||
for (auto i = goalMap.begin(); i != goalMap.end();) {
|
||||
if (!removeGoal(goal, i->second))
|
||||
i = goalMap.erase(i);
|
||||
else ++i;
|
||||
else
|
||||
++i;
|
||||
}
|
||||
return !goalMap.empty();
|
||||
}
|
||||
|
||||
|
||||
template<typename K, typename G>
|
||||
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
|
||||
template<typename G>
|
||||
static bool removeGoal(std::shared_ptr<G> goal, typename DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<G>>>::ChildNode & node)
|
||||
{
|
||||
/* !!! inefficient */
|
||||
cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
|
||||
return gp.lock() != goal;
|
||||
});
|
||||
}
|
||||
|
||||
template<typename K>
|
||||
static void removeGoal(std::shared_ptr<DerivationGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<DerivationGoal>>::ChildNode> & goalMap);
|
||||
|
||||
template<typename K>
|
||||
static void removeGoal(std::shared_ptr<DerivationGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<DerivationGoal>>::ChildNode> & goalMap)
|
||||
{
|
||||
/* !!! inefficient */
|
||||
cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<DerivationGoal>>::ChildNode & node) -> bool {
|
||||
if (node.value.lock() == goal)
|
||||
node.value.reset();
|
||||
removeGoal(goal, node.childMap);
|
||||
return !node.value.expired() || !node.childMap.empty();
|
||||
});
|
||||
return removeGoal(goal, node.value) || removeGoal(goal, node.childMap);
|
||||
}
|
||||
|
||||
|
||||
void Worker::removeGoal(GoalPtr goal)
|
||||
{
|
||||
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
|
||||
nix::removeGoal(drvGoal, derivationGoals.map);
|
||||
if (auto drvGoal = std::dynamic_pointer_cast<DerivationTrampolineGoal>(goal))
|
||||
nix::removeGoal(drvGoal, derivationTrampolineGoals.map);
|
||||
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
|
||||
nix::removeGoal(drvGoal, derivationGoals);
|
||||
else if (auto drvBuildingGoal = std::dynamic_pointer_cast<DerivationBuildingGoal>(goal))
|
||||
nix::removeGoal(drvBuildingGoal, derivationBuildingGoals);
|
||||
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
||||
|
|
@ -312,21 +307,18 @@ void Worker::run(const Goals & _topGoals)
|
|||
|
||||
for (auto & i : _topGoals) {
|
||||
topGoals.insert(i);
|
||||
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
|
||||
if (auto goal = dynamic_cast<DerivationTrampolineGoal *>(i.get())) {
|
||||
topPaths.push_back(DerivedPath::Built {
|
||||
.drvPath = goal->drvReq,
|
||||
.outputs = goal->wantedOutputs,
|
||||
});
|
||||
} else
|
||||
if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
|
||||
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
|
||||
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
|
||||
}
|
||||
}
|
||||
|
||||
/* Call queryMissing() to efficiently query substitutes. */
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
uint64_t downloadSize, narSize;
|
||||
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
store.queryMissing(topPaths);
|
||||
|
||||
debug("entered goal loop");
|
||||
|
||||
|
|
|
|||
|
|
@ -949,14 +949,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
case WorkerProto::Op::QueryMissing: {
|
||||
auto targets = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
uint64_t downloadSize, narSize;
|
||||
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
auto missing = store->queryMissing(targets);
|
||||
logger->stopWork();
|
||||
WorkerProto::write(*store, wconn, willBuild);
|
||||
WorkerProto::write(*store, wconn, willSubstitute);
|
||||
WorkerProto::write(*store, wconn, unknown);
|
||||
conn.to << downloadSize << narSize;
|
||||
WorkerProto::write(*store, wconn, missing.willBuild);
|
||||
WorkerProto::write(*store, wconn, missing.willSubstitute);
|
||||
WorkerProto::write(*store, wconn, missing.unknown);
|
||||
conn.to << missing.downloadSize << missing.narSize;
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
|
|||
|
||||
// instantiations
|
||||
|
||||
#include "nix/store/build/derivation-goal.hh"
|
||||
#include "nix/store/build/derivation-trampoline-goal.hh"
|
||||
namespace nix {
|
||||
|
||||
template<>
|
||||
|
|
@ -69,7 +69,7 @@ std::strong_ordering DerivedPathMap<StringSet>::ChildNode::operator <=> (
|
|||
template struct DerivedPathMap<StringSet>::ChildNode;
|
||||
template struct DerivedPathMap<StringSet>;
|
||||
|
||||
template struct DerivedPathMap<std::weak_ptr<DerivationGoal>>;
|
||||
template struct DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<DerivationTrampolineGoal>>>;
|
||||
|
||||
|
||||
};
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ std::vector<Path> getUserConfigFiles()
|
|||
return files;
|
||||
}
|
||||
|
||||
unsigned int Settings::getDefaultCores()
|
||||
unsigned int Settings::getDefaultCores() const
|
||||
{
|
||||
const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency());
|
||||
const unsigned int maxCPU = getMaxCPU();
|
||||
|
|
|
|||
|
|
@ -29,7 +29,9 @@ void runPostBuildHook(
|
|||
const StorePathSet & outputPaths);
|
||||
|
||||
/**
|
||||
* A goal for building some or all of the outputs of a derivation.
|
||||
* A goal for building a derivation. Substitution, (or any other method of
|
||||
* obtaining the outputs) will not be attempted, so it is the calling goal's
|
||||
* responsibility to try to substitute first.
|
||||
*/
|
||||
struct DerivationBuildingGoal : public Goal
|
||||
{
|
||||
|
|
|
|||
|
|
@ -22,47 +22,33 @@ void runPostBuildHook(
|
|||
const StorePathSet & outputPaths);
|
||||
|
||||
/**
|
||||
* A goal for building some or all of the outputs of a derivation.
|
||||
* A goal for realising a single output of a derivation. Various sorts of
|
||||
* fetching (which will be done by other goal types) is tried, and if none of
|
||||
* those succeed, the derivation is attempted to be built.
|
||||
*
|
||||
* This is a purely "administrative" goal type, which doesn't do any
|
||||
* "real work" of substituting (that would be `PathSubstitutionGoal` or
|
||||
* `DrvOutputSubstitutionGoal`) or building (that would be a
|
||||
* `DerivationBuildingGoal`). This goal type creates those types of
|
||||
* goals to attempt each way of realisation a derivation; they are tried
|
||||
* sequentially in order of preference.
|
||||
*
|
||||
* The derivation must already be gotten (in memory, in C++, parsed) and passed
|
||||
* to the caller. If the derivation itself needs to be gotten first, a
|
||||
* `DerivationTrampolineGoal` goal must be used instead.
|
||||
*/
|
||||
struct DerivationGoal : public Goal
|
||||
{
|
||||
/** The path of the derivation. */
|
||||
ref<const SingleDerivedPath> drvReq;
|
||||
StorePath drvPath;
|
||||
|
||||
/**
|
||||
* The specific outputs that we need to build.
|
||||
*/
|
||||
OutputsSpec wantedOutputs;
|
||||
OutputName wantedOutput;
|
||||
|
||||
/**
|
||||
* See `needRestart`; just for that field.
|
||||
*/
|
||||
enum struct NeedRestartForMoreOutputs {
|
||||
/**
|
||||
* The goal state machine is progressing based on the current value of
|
||||
* `wantedOutputs. No actions are needed.
|
||||
*/
|
||||
OutputsUnmodifiedDontNeed,
|
||||
/**
|
||||
* `wantedOutputs` has been extended, but the state machine is
|
||||
* proceeding according to its old value, so we need to restart.
|
||||
*/
|
||||
OutputsAddedDoNeed,
|
||||
/**
|
||||
* The goal state machine has progressed to the point of doing a build,
|
||||
* in which case all outputs will be produced, so extensions to
|
||||
* `wantedOutputs` no longer require a restart.
|
||||
*/
|
||||
BuildInProgressWillNotNeed,
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether additional wanted outputs have been added.
|
||||
*/
|
||||
NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifiedDontNeed;
|
||||
|
||||
/**
|
||||
* The derivation stored at `drvReq`.
|
||||
* The derivation stored at drvPath.
|
||||
*/
|
||||
std::unique_ptr<Derivation> drv;
|
||||
|
||||
|
|
@ -76,11 +62,8 @@ struct DerivationGoal : public Goal
|
|||
|
||||
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds;
|
||||
|
||||
DerivationGoal(ref<const SingleDerivedPath> drvReq,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const OutputsSpec & wantedOutputs, Worker & worker,
|
||||
DerivationGoal(const StorePath & drvPath, const Derivation & drv,
|
||||
const OutputName & wantedOutput, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
~DerivationGoal() = default;
|
||||
|
||||
|
|
@ -88,24 +71,18 @@ struct DerivationGoal : public Goal
|
|||
|
||||
std::string key() override;
|
||||
|
||||
/**
|
||||
* Add wanted outputs to an already existing derivation goal.
|
||||
*/
|
||||
void addWantedOutputs(const OutputsSpec & outputs);
|
||||
|
||||
/**
|
||||
* The states.
|
||||
*/
|
||||
Co loadDerivation();
|
||||
Co haveDerivation(StorePath drvPath);
|
||||
Co haveDerivation();
|
||||
|
||||
/**
|
||||
* Wrappers around the corresponding Store methods that first consult the
|
||||
* derivation. This is currently needed because when there is no drv file
|
||||
* there also is no DB entry.
|
||||
*/
|
||||
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & drvPath);
|
||||
OutputPathMap queryDerivationOutputMap(const StorePath & drvPath);
|
||||
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
|
||||
OutputPathMap queryDerivationOutputMap();
|
||||
|
||||
/**
|
||||
* Update 'initialOutputs' to determine the current status of the
|
||||
|
|
@ -113,18 +90,17 @@ struct DerivationGoal : public Goal
|
|||
* whether all outputs are valid and non-corrupt, and a
|
||||
* 'SingleDrvOutputs' structure containing the valid outputs.
|
||||
*/
|
||||
std::pair<bool, SingleDrvOutputs> checkPathValidity(const StorePath & drvPath);
|
||||
std::pair<bool, SingleDrvOutputs> checkPathValidity();
|
||||
|
||||
/**
|
||||
* Aborts if any output is not valid or corrupt, and otherwise
|
||||
* returns a 'SingleDrvOutputs' structure containing all outputs.
|
||||
*/
|
||||
SingleDrvOutputs assertPathValidity(const StorePath & drvPath);
|
||||
SingleDrvOutputs assertPathValidity();
|
||||
|
||||
Co repairClosure(StorePath drvPath);
|
||||
Co repairClosure();
|
||||
|
||||
Done done(
|
||||
const StorePath & drvPath,
|
||||
BuildResult::Status status,
|
||||
SingleDrvOutputs builtOutputs = {},
|
||||
std::optional<Error> ex = {});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/store/parsed-derivations.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/store/pathlocks.hh"
|
||||
#include "nix/store/build/goal.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* This is the "outermost" goal type relating to derivations --- by that
|
||||
* we mean that this one calls all the others for a given derivation.
|
||||
*
|
||||
* This is a purely "administrative" goal type, which doesn't do any "real
|
||||
* work". See `DerivationGoal` for what we mean by such an administrative goal.
|
||||
*
|
||||
* # Rationale
|
||||
*
|
||||
* It exists to solve two problems:
|
||||
*
|
||||
* 1. We want to build a derivation we don't yet have.
|
||||
*
|
||||
* Traditionally, that simply means we try to substitute the missing
|
||||
* derivation; simple enough. However, with (currently experimental)
|
||||
* dynamic derivations, derivations themselves can be the outputs of
|
||||
* other derivations. That means the general case is that a
|
||||
* `DerivationTrampolineGoal` needs to create *another*
|
||||
* `DerivationTrampolineGoal` goal to realize the derivation it needs.
|
||||
* That goal in turn might need to create a third
|
||||
* `DerivationTrampolineGoal`, the induction down to a statically known
|
||||
* derivation as the base case is arbitrary deep.
|
||||
*
|
||||
* 2. Only a subset of outputs is needed, but such subsets are discovered
|
||||
* dynamically.
|
||||
*
|
||||
* Consider derivations:
|
||||
*
|
||||
* - A has outputs x, y, and z
|
||||
*
|
||||
* - B needs A^x,y
|
||||
*
|
||||
* - C needs A^y,z and B's single output
|
||||
*
|
||||
* With the current `Worker` architecture, we're first discover
|
||||
* needing `A^y,z` and then discover needing `A^x,y`. Of course, we
|
||||
* don't want to download `A^y` twice, either.
|
||||
*
|
||||
* The way we handle sharing work for `A^y` is to have
|
||||
* `DerivationGoal` just handle a single output, and do slightly more
|
||||
* work (though it is just an "administrative" goal too), and
|
||||
* `DerivationTrampolineGoal` handle sets of goals, but have it (once the
|
||||
* derivation itself has been gotten) *just* create
|
||||
* `DerivationGoal`s.
|
||||
*
|
||||
* That means it is fine to create man `DerivationTrampolineGoal` with
|
||||
* overlapping sets of outputs, because all the "real work" will be
|
||||
* coordinated via `DerivationGoal`s, and sharing will be discovered.
|
||||
*
|
||||
* Both these problems *can* be solved by having just a more powerful
|
||||
* `DerivationGoal`, but that makes `DerivationGoal` more complex.
|
||||
* However the more complex `DerivationGoal` has these downsides:
|
||||
*
|
||||
* 1. It needs to cope with only sometimes knowing a `StorePath drvPath`
|
||||
* (as opposed to a more general `SingleDerivedPath drvPath` with will
|
||||
* be only resolved to a `StorePath` part way through the control flow).
|
||||
*
|
||||
* 2. It needs complicated "restarting logic" to cope with the set of
|
||||
* "wanted outputs" growing over time.
|
||||
*
|
||||
* (1) is not so bad, but (2) is quite scary, and has been a source of
|
||||
* bugs in the past. By splitting out `DerivationTrampolineGoal`, we
|
||||
* crucially avoid a need for (2), letting goal sharing rather than
|
||||
* ad-hoc retry mechanisms accomplish the deduplication we need. Solving
|
||||
* (1) is just a by-product and extra bonus of creating
|
||||
* `DerivationTrampolineGoal`.
|
||||
*
|
||||
* # Misc Notes
|
||||
*
|
||||
* If we already have the derivation (e.g. if the evaluator has created
|
||||
* the derivation locally and then instructed the store to build it), we
|
||||
* can skip the derivation-getting goal entirely as a small
|
||||
* optimization.
|
||||
*/
|
||||
struct DerivationTrampolineGoal : public Goal
|
||||
{
|
||||
/**
|
||||
* How to obtain a store path of the derivation to build.
|
||||
*/
|
||||
ref<const SingleDerivedPath> drvReq;
|
||||
|
||||
/**
|
||||
* The specific outputs that we need to build.
|
||||
*/
|
||||
OutputsSpec wantedOutputs;
|
||||
|
||||
DerivationTrampolineGoal(
|
||||
ref<const SingleDerivedPath> drvReq,
|
||||
const OutputsSpec & wantedOutputs,
|
||||
Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
|
||||
DerivationTrampolineGoal(
|
||||
const StorePath & drvPath,
|
||||
const OutputsSpec & wantedOutputs,
|
||||
const Derivation & drv,
|
||||
Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
|
||||
virtual ~DerivationTrampolineGoal();
|
||||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
std::string key() override;
|
||||
|
||||
JobCategory jobCategory() const override
|
||||
{
|
||||
return JobCategory::Administration;
|
||||
};
|
||||
|
||||
private:
|
||||
|
||||
BuildMode buildMode;
|
||||
|
||||
Co init();
|
||||
Co haveDerivation(StorePath drvPath, Derivation drv);
|
||||
|
||||
/**
|
||||
* Shared between both constructors
|
||||
*/
|
||||
void commonInit();
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -105,13 +105,11 @@ public:
|
|||
*/
|
||||
ExitCode exitCode = ecBusy;
|
||||
|
||||
protected:
|
||||
/**
|
||||
* Build result.
|
||||
*/
|
||||
BuildResult buildResult;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Suspend our goal and wait until we get `work`-ed again.
|
||||
* `co_await`-able by @ref Co.
|
||||
|
|
@ -358,18 +356,6 @@ protected:
|
|||
public:
|
||||
virtual void cleanup() { }
|
||||
|
||||
/**
|
||||
* Project a `BuildResult` with just the information that pertains
|
||||
* to the given request.
|
||||
*
|
||||
* In general, goals may be aliased between multiple requests, and
|
||||
* the stored `BuildResult` has information for the union of all
|
||||
* requests. We don't want to leak what the other request are for
|
||||
* sake of both privacy and determinism, and this "safe accessor"
|
||||
* ensures we don't.
|
||||
*/
|
||||
BuildResult getBuildResult(const DerivedPath &) const;
|
||||
|
||||
/**
|
||||
* Hack to say that this goal should not log `ex`, but instead keep
|
||||
* it around. Set by a waitee which sees itself as the designated
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
namespace nix {
|
||||
|
||||
/* Forward definition. */
|
||||
struct DerivationTrampolineGoal;
|
||||
struct DerivationGoal;
|
||||
struct DerivationBuildingGoal;
|
||||
struct PathSubstitutionGoal;
|
||||
|
|
@ -33,6 +34,7 @@ class DrvOutputSubstitutionGoal;
|
|||
*/
|
||||
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
|
||||
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
|
||||
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
|
||||
|
||||
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
||||
|
||||
|
|
@ -106,8 +108,9 @@ private:
|
|||
* same derivation / path.
|
||||
*/
|
||||
|
||||
DerivedPathMap<std::weak_ptr<DerivationGoal>> derivationGoals;
|
||||
DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<DerivationTrampolineGoal>>> derivationTrampolineGoals;
|
||||
|
||||
std::map<StorePath, std::map<OutputName, std::weak_ptr<DerivationGoal>>> derivationGoals;
|
||||
std::map<StorePath, std::weak_ptr<DerivationBuildingGoal>> derivationBuildingGoals;
|
||||
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
|
||||
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
|
||||
|
|
@ -204,16 +207,20 @@ private:
|
|||
template<class G, typename... Args>
|
||||
std::shared_ptr<G> initGoalIfNeeded(std::weak_ptr<G> & goal_weak, Args && ...args);
|
||||
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
|
||||
ref<const SingleDerivedPath> drvReq, const OutputsSpec & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
|
||||
public:
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoal(
|
||||
std::shared_ptr<DerivationTrampolineGoal> makeDerivationTrampolineGoal(
|
||||
ref<const SingleDerivedPath> drvReq,
|
||||
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
|
||||
const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
|
||||
public:
|
||||
std::shared_ptr<DerivationTrampolineGoal> makeDerivationTrampolineGoal(
|
||||
const StorePath & drvPath,
|
||||
const OutputsSpec & wantedOutputs,
|
||||
const Derivation & drv,
|
||||
BuildMode buildMode = bmNormal);
|
||||
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoal(
|
||||
const StorePath & drvPath, const Derivation & drv,
|
||||
const OutputName & wantedOutput, BuildMode buildMode = bmNormal);
|
||||
|
||||
/**
|
||||
* @ref DerivationBuildingGoal "derivation goal"
|
||||
|
|
@ -232,7 +239,7 @@ public:
|
|||
* Make a goal corresponding to the `DerivedPath`.
|
||||
*
|
||||
* It will be a `DerivationGoal` for a `DerivedPath::Built` or
|
||||
* a `SubstitutionGoal` for a `DerivedPath::Opaque`.
|
||||
* a `PathSubstitutionGoal` for a `DerivedPath::Opaque`.
|
||||
*/
|
||||
GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal);
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ namespace nix {
|
|||
* @param V A type to instantiate for each output. It should probably
|
||||
* should be an "optional" type so not every interior node has to have a
|
||||
* value. For example, the scheduler uses
|
||||
* `DerivedPathMap<std::weak_ptr<DerivationCreationAndRealisationGoal>>` to
|
||||
* `DerivedPathMap<std::weak_ptr<DerivationTrampolineGoal>>` to
|
||||
* remember which goals correspond to which outputs. `* const Something`
|
||||
* or `std::optional<Something>` would also be good choices for
|
||||
* "optional" types.
|
||||
|
|
|
|||
|
|
@ -43,8 +43,6 @@ const uint32_t maxIdsPerBuild =
|
|||
|
||||
class Settings : public Config {
|
||||
|
||||
unsigned int getDefaultCores();
|
||||
|
||||
StringSet getDefaultSystemFeatures();
|
||||
|
||||
StringSet getDefaultExtraPlatforms();
|
||||
|
|
@ -57,6 +55,8 @@ public:
|
|||
|
||||
Settings();
|
||||
|
||||
unsigned int getDefaultCores() const;
|
||||
|
||||
Path nixPrefix;
|
||||
|
||||
/**
|
||||
|
|
@ -153,7 +153,7 @@ public:
|
|||
|
||||
Setting<unsigned int> buildCores{
|
||||
this,
|
||||
getDefaultCores(),
|
||||
0,
|
||||
"cores",
|
||||
R"(
|
||||
Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation.
|
||||
|
|
@ -166,15 +166,13 @@ public:
|
|||
-->
|
||||
For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it passes the `-j${NIX_BUILD_CORES}` flag to GNU Make.
|
||||
|
||||
The value `0` means that the `builder` should use all available CPU cores in the system.
|
||||
If set to `0`, nix will detect the number of CPU cores and pass this number via NIX_BUILD_CORES.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting.
|
||||
)",
|
||||
{"build-cores"},
|
||||
// Don't document the machine-specific default value
|
||||
false};
|
||||
{"build-cores"}};
|
||||
|
||||
/**
|
||||
* Read-only mode. Don't copy stuff to the store, don't change
|
||||
|
|
@ -873,7 +871,7 @@ public:
|
|||
On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available.
|
||||
This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn].
|
||||
|
||||
[nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix.
|
||||
[nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix
|
||||
|
||||
Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled.
|
||||
)",
|
||||
|
|
|
|||
|
|
@ -435,7 +435,6 @@ private:
|
|||
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
|
||||
|
||||
friend struct PathSubstitutionGoal;
|
||||
friend struct SubstitutionGoal;
|
||||
friend struct DerivationGoal;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ headers = [config_pub_h] + files(
|
|||
'build/derivation-goal.hh',
|
||||
'build/derivation-building-goal.hh',
|
||||
'build/derivation-building-misc.hh',
|
||||
'build/derivation-trampoline-goal.hh',
|
||||
'build/drv-output-substitution-goal.hh',
|
||||
'build/goal.hh',
|
||||
'build/substitution-goal.hh',
|
||||
|
|
|
|||
|
|
@ -149,9 +149,7 @@ struct RemoteStore :
|
|||
|
||||
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
|
||||
|
||||
void queryMissing(const std::vector<DerivedPath> & targets,
|
||||
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
|
||||
uint64_t & downloadSize, uint64_t & narSize) override;
|
||||
MissingPaths queryMissing(const std::vector<DerivedPath> & targets) override;
|
||||
|
||||
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
|
||||
|
||||
|
|
|
|||
|
|
@ -71,6 +71,18 @@ struct KeyedBuildResult;
|
|||
|
||||
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
|
||||
|
||||
/**
|
||||
* Information about what paths will be built or substituted, returned
|
||||
* by Store::queryMissing().
|
||||
*/
|
||||
struct MissingPaths
|
||||
{
|
||||
StorePathSet willBuild;
|
||||
StorePathSet willSubstitute;
|
||||
StorePathSet unknown;
|
||||
uint64_t downloadSize{0};
|
||||
uint64_t narSize{0};
|
||||
};
|
||||
|
||||
/**
|
||||
* About the class hierarchy of the store types:
|
||||
|
|
@ -694,9 +706,7 @@ public:
|
|||
* derivations that will be built, and the set of output paths that
|
||||
* will be substituted.
|
||||
*/
|
||||
virtual void queryMissing(const std::vector<DerivedPath> & targets,
|
||||
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
|
||||
uint64_t & downloadSize, uint64_t & narSize);
|
||||
virtual MissingPaths queryMissing(const std::vector<DerivedPath> & targets);
|
||||
|
||||
/**
|
||||
* Sort a set of paths topologically under the references
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ struct LocalBinaryCacheStore :
|
|||
, BinaryCacheStore{*config}
|
||||
, config{config}
|
||||
{
|
||||
init();
|
||||
}
|
||||
|
||||
void init() override;
|
||||
|
|
@ -126,10 +125,12 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes()
|
|||
}
|
||||
|
||||
ref<Store> LocalBinaryCacheStoreConfig::openStore() const {
|
||||
return make_ref<LocalBinaryCacheStore>(ref{
|
||||
auto store = make_ref<LocalBinaryCacheStore>(ref{
|
||||
// FIXME we shouldn't actually need a mutable config
|
||||
std::const_pointer_cast<LocalBinaryCacheStore::Config>(shared_from_this())
|
||||
});
|
||||
store->init();
|
||||
return store;
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<LocalBinaryCacheStore::Config> regLocalBinaryCacheStore;
|
||||
|
|
|
|||
|
|
@ -255,6 +255,7 @@ sources = files(
|
|||
'build-result.cc',
|
||||
'build/derivation-goal.cc',
|
||||
'build/derivation-building-goal.cc',
|
||||
'build/derivation-trampoline-goal.cc',
|
||||
'build/drv-output-substitution-goal.cc',
|
||||
'build/entry-points.cc',
|
||||
'build/goal.cc',
|
||||
|
|
|
|||
|
|
@ -98,23 +98,17 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv)
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
||||
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
|
||||
uint64_t & downloadSize_, uint64_t & narSize_)
|
||||
MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
|
||||
{
|
||||
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
|
||||
|
||||
downloadSize_ = narSize_ = 0;
|
||||
|
||||
// FIXME: make async.
|
||||
ThreadPool pool(fileTransferSettings.httpConnections);
|
||||
|
||||
struct State
|
||||
{
|
||||
std::unordered_set<std::string> done;
|
||||
StorePathSet & unknown, & willSubstitute, & willBuild;
|
||||
uint64_t & downloadSize;
|
||||
uint64_t & narSize;
|
||||
MissingPaths res;
|
||||
};
|
||||
|
||||
struct DrvState
|
||||
|
|
@ -125,7 +119,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
DrvState(size_t left) : left(left) { }
|
||||
};
|
||||
|
||||
Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
|
||||
Sync<State> state_;
|
||||
|
||||
std::function<void(DerivedPath)> doPath;
|
||||
|
||||
|
|
@ -143,7 +137,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
|
||||
{
|
||||
auto state(state_.lock());
|
||||
state->willBuild.insert(drvPath);
|
||||
state->res.willBuild.insert(drvPath);
|
||||
}
|
||||
|
||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) {
|
||||
|
|
@ -203,7 +197,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
if (!isValidPath(drvPath)) {
|
||||
// FIXME: we could try to substitute the derivation.
|
||||
auto state(state_.lock());
|
||||
state->unknown.insert(drvPath);
|
||||
state->res.unknown.insert(drvPath);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -282,7 +276,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
|
||||
if (infos.empty()) {
|
||||
auto state(state_.lock());
|
||||
state->unknown.insert(bo.path);
|
||||
state->res.unknown.insert(bo.path);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -291,9 +285,9 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
|
||||
{
|
||||
auto state(state_.lock());
|
||||
state->willSubstitute.insert(bo.path);
|
||||
state->downloadSize += info->second.downloadSize;
|
||||
state->narSize += info->second.narSize;
|
||||
state->res.willSubstitute.insert(bo.path);
|
||||
state->res.downloadSize += info->second.downloadSize;
|
||||
state->res.narSize += info->second.narSize;
|
||||
}
|
||||
|
||||
for (auto & ref : info->second.references)
|
||||
|
|
@ -306,6 +300,8 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
pool.enqueue(std::bind(doPath, path));
|
||||
|
||||
pool.process();
|
||||
|
||||
return std::move(state_.lock()->res);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -855,9 +855,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
|
|||
}
|
||||
|
||||
|
||||
void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
|
||||
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
|
||||
uint64_t & downloadSize, uint64_t & narSize)
|
||||
MissingPaths RemoteStore::queryMissing(const std::vector<DerivedPath> & targets)
|
||||
{
|
||||
{
|
||||
auto conn(getConnection());
|
||||
|
|
@ -868,16 +866,16 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
|
|||
conn->to << WorkerProto::Op::QueryMissing;
|
||||
WorkerProto::write(*this, *conn, targets);
|
||||
conn.processStderr();
|
||||
willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
unknown = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
conn->from >> downloadSize >> narSize;
|
||||
return;
|
||||
MissingPaths res;
|
||||
res.willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
res.willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
res.unknown = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
conn->from >> res.downloadSize >> res.narSize;
|
||||
return res;
|
||||
}
|
||||
|
||||
fallback:
|
||||
return Store::queryMissing(targets, willBuild, willSubstitute,
|
||||
unknown, downloadSize, narSize);
|
||||
return Store::queryMissing(targets);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -143,13 +143,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor
|
|||
unsupported("addSignatures");
|
||||
}
|
||||
|
||||
void queryMissing(
|
||||
const std::vector<DerivedPath> & targets,
|
||||
StorePathSet & willBuild,
|
||||
StorePathSet & willSubstitute,
|
||||
StorePathSet & unknown,
|
||||
uint64_t & downloadSize,
|
||||
uint64_t & narSize) override;
|
||||
MissingPaths queryMissing(const std::vector<DerivedPath> & targets) override;
|
||||
|
||||
virtual std::optional<std::string> getBuildLogExact(const StorePath & path) override
|
||||
{
|
||||
|
|
@ -306,19 +300,14 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
|
|||
return results;
|
||||
}
|
||||
|
||||
void RestrictedStore::queryMissing(
|
||||
const std::vector<DerivedPath> & targets,
|
||||
StorePathSet & willBuild,
|
||||
StorePathSet & willSubstitute,
|
||||
StorePathSet & unknown,
|
||||
uint64_t & downloadSize,
|
||||
uint64_t & narSize)
|
||||
MissingPaths RestrictedStore::queryMissing(const std::vector<DerivedPath> & targets)
|
||||
{
|
||||
/* This is slightly impure since it leaks information to the
|
||||
client about what paths will be built/substituted or are
|
||||
already present. Probably not a big deal. */
|
||||
|
||||
std::vector<DerivedPath> allowed;
|
||||
StorePathSet unknown;
|
||||
for (auto & req : targets) {
|
||||
if (goal.isAllowed(req))
|
||||
allowed.emplace_back(req);
|
||||
|
|
@ -326,7 +315,12 @@ void RestrictedStore::queryMissing(
|
|||
unknown.insert(pathPartOfReq(req));
|
||||
}
|
||||
|
||||
next->queryMissing(allowed, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
auto res = next->queryMissing(allowed);
|
||||
|
||||
for (auto & p : unknown)
|
||||
res.unknown.insert(p);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,10 +37,11 @@ namespace nix {
|
|||
struct S3Error : public Error
|
||||
{
|
||||
Aws::S3::S3Errors err;
|
||||
Aws::String exceptionName;
|
||||
|
||||
template<typename... Args>
|
||||
S3Error(Aws::S3::S3Errors err, const Args & ... args)
|
||||
: Error(args...), err(err) { };
|
||||
S3Error(Aws::S3::S3Errors err, Aws::String exceptionName, const Args & ... args)
|
||||
: Error(args...), err(err), exceptionName(exceptionName) { };
|
||||
};
|
||||
|
||||
/* Helper: given an Outcome<R, E>, return R in case of success, or
|
||||
|
|
@ -51,6 +52,7 @@ R && checkAws(std::string_view s, Aws::Utils::Outcome<R, E> && outcome)
|
|||
if (!outcome.IsSuccess())
|
||||
throw S3Error(
|
||||
outcome.GetError().GetErrorType(),
|
||||
outcome.GetError().GetExceptionName(),
|
||||
fmt(
|
||||
"%s: %s (request id: %s)",
|
||||
s,
|
||||
|
|
@ -226,7 +228,13 @@ S3Helper::FileTransferResult S3Helper::getObject(
|
|||
|
||||
} catch (S3Error & e) {
|
||||
if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) &&
|
||||
(e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw;
|
||||
(e.err != Aws::S3::S3Errors::ACCESS_DENIED) &&
|
||||
// Expired tokens are not really an error, more of a caching problem. Should be treated same as 403.
|
||||
//
|
||||
// AWS unwilling to provide a specific error type for the situation (https://github.com/aws/aws-sdk-cpp/issues/1843)
|
||||
// so use this hack
|
||||
(e.exceptionName != "ExpiredToken")
|
||||
) throw;
|
||||
}
|
||||
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
|
|
@ -281,8 +289,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
|
|||
, s3Helper(config->profile, config->region, config->scheme, config->endpoint)
|
||||
{
|
||||
diskCache = getNarInfoDiskCache();
|
||||
|
||||
init();
|
||||
}
|
||||
|
||||
std::string getUri() override
|
||||
|
|
@ -334,6 +340,10 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
|
|||
auto & error = res.GetError();
|
||||
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
|
||||
|| error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
|
||||
// Expired tokens are not really an error, more of a caching problem. Should be treated same as 403.
|
||||
// AWS unwilling to provide a specific error type for the situation (https://github.com/aws/aws-sdk-cpp/issues/1843)
|
||||
// so use this hack
|
||||
|| (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN && error.GetExceptionName() == "ExpiredToken")
|
||||
// If bucket listing is disabled, 404s turn into 403s
|
||||
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
|
||||
return false;
|
||||
|
|
@ -585,10 +595,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
|
|||
|
||||
ref<Store> S3BinaryCacheStoreImpl::Config::openStore() const
|
||||
{
|
||||
return make_ref<S3BinaryCacheStoreImpl>(ref{
|
||||
auto store = make_ref<S3BinaryCacheStoreImpl>(ref{
|
||||
// FIXME we shouldn't actually need a mutable config
|
||||
std::const_pointer_cast<S3BinaryCacheStore::Config>(shared_from_this())
|
||||
});
|
||||
store->init();
|
||||
return store;
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<S3BinaryCacheStoreImpl::Config> regS3BinaryCacheStore;
|
||||
|
|
|
|||
|
|
@ -250,7 +250,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning)
|
|||
if (now > nextWarning) {
|
||||
nextWarning = now + 10;
|
||||
logWarning({
|
||||
.msg = HintFmt(e.what())
|
||||
.msg = e.info().msg
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -790,15 +790,12 @@ void Store::substitutePaths(const StorePathSet & paths)
|
|||
for (auto & path : paths)
|
||||
if (!path.isDerivation())
|
||||
paths2.emplace_back(DerivedPath::Opaque{path});
|
||||
uint64_t downloadSize, narSize;
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
queryMissing(paths2,
|
||||
willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
auto missing = queryMissing(paths2);
|
||||
|
||||
if (!willSubstitute.empty())
|
||||
if (!missing.willSubstitute.empty())
|
||||
try {
|
||||
std::vector<DerivedPath> subs;
|
||||
for (auto & p : willSubstitute) subs.emplace_back(DerivedPath::Opaque{p});
|
||||
for (auto & p : missing.willSubstitute) subs.emplace_back(DerivedPath::Opaque{p});
|
||||
buildPaths(subs);
|
||||
} catch (Error & e) {
|
||||
logWarning(e.info());
|
||||
|
|
|
|||
|
|
@ -160,6 +160,8 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl
|
|||
|
||||
if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
|
||||
Strings sandboxArgs;
|
||||
sandboxArgs.push_back("_NIX_BUILD_TOP");
|
||||
sandboxArgs.push_back(tmpDir);
|
||||
sandboxArgs.push_back("_GLOBAL_TMP_DIR");
|
||||
sandboxArgs.push_back(globalTmpDir);
|
||||
if (drvOptions.allowLocalNetworking) {
|
||||
|
|
|
|||
|
|
@ -1083,7 +1083,7 @@ void DerivationBuilderImpl::initEnv()
|
|||
env["NIX_STORE"] = store.storeDir;
|
||||
|
||||
/* The maximum number of cores to utilize for parallel building. */
|
||||
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores);
|
||||
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
|
||||
|
||||
/* In non-structured mode, set all bindings either directory in the
|
||||
environment or via a file, as specified by
|
||||
|
|
|
|||
|
|
@ -368,6 +368,13 @@ struct ChrootLinuxDerivationBuilder : LinuxDerivationBuilder
|
|||
if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
|
||||
throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
|
||||
|
||||
pathsInChroot = getPathsInSandbox();
|
||||
|
||||
for (auto & i : inputPaths) {
|
||||
auto p = store.printStorePath(i);
|
||||
pathsInChroot.insert_or_assign(p, store.toRealPath(p));
|
||||
}
|
||||
|
||||
/* If we're repairing, checking or rebuilding part of a
|
||||
multiple-outputs derivation, it's possible that we're
|
||||
rebuilding a path that is in settings.sandbox-paths
|
||||
|
|
@ -391,13 +398,6 @@ struct ChrootLinuxDerivationBuilder : LinuxDerivationBuilder
|
|||
chownToBuilder(*cgroup + "/cgroup.threads");
|
||||
// chownToBuilder(*cgroup + "/cgroup.subtree_control");
|
||||
}
|
||||
|
||||
pathsInChroot = getPathsInSandbox();
|
||||
|
||||
for (auto & i : inputPaths) {
|
||||
auto p = store.printStorePath(i);
|
||||
pathsInChroot.insert_or_assign(p, store.toRealPath(p));
|
||||
}
|
||||
}
|
||||
|
||||
Strings getPreBuildHookArgs() override
|
||||
|
|
|
|||
|
|
@ -29,12 +29,14 @@ R""(
|
|||
; Allow getpwuid.
|
||||
(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo"))
|
||||
|
||||
; Access to /tmp.
|
||||
; Access to /tmp and the build directory.
|
||||
; The network-outbound/network-inbound ones are for unix domain sockets, which
|
||||
; we allow access to in TMPDIR (but if we allow them more broadly, you could in
|
||||
; theory escape the sandbox)
|
||||
(allow file* process-exec network-outbound network-inbound
|
||||
(literal "/tmp") (subpath TMPDIR))
|
||||
(literal "/tmp")
|
||||
(subpath TMPDIR)
|
||||
(subpath (param "_NIX_BUILD_TOP")))
|
||||
|
||||
; Some packages like to read the system version.
|
||||
(allow file-read*
|
||||
|
|
|
|||
|
|
@ -197,7 +197,7 @@ bool useBuildUsers()
|
|||
#ifdef __linux__
|
||||
static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && isRootUser();
|
||||
return b;
|
||||
#elif defined(__APPLE__) && defined(__FreeBSD__)
|
||||
#elif defined(__APPLE__) || defined(__FreeBSD__)
|
||||
static bool b = settings.buildUsersGroup != "" && isRootUser();
|
||||
return b;
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ boost = dependency(
|
|||
'boost',
|
||||
modules : ['context', 'coroutine', 'iostreams'],
|
||||
include_type: 'system',
|
||||
version: '>=1.82.0'
|
||||
)
|
||||
# boost is a public dependency, but not a pkg-config dependency unfortunately, so we
|
||||
# put in `deps_other`.
|
||||
|
|
|
|||
|
|
@ -194,10 +194,6 @@ size_t StringSource::read(char * data, size_t len)
|
|||
}
|
||||
|
||||
|
||||
#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600
|
||||
#error Coroutines are broken in this version of Boost!
|
||||
#endif
|
||||
|
||||
std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
|
||||
{
|
||||
struct SourceToSink : FinishSink
|
||||
|
|
|
|||
|
|
@ -190,8 +190,10 @@ void ignoreExceptionInDestructor(Verbosity lvl)
|
|||
try {
|
||||
try {
|
||||
throw;
|
||||
} catch (Error & e) {
|
||||
printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg);
|
||||
} catch (std::exception & e) {
|
||||
printMsg(lvl, "error (ignored): %1%", e.what());
|
||||
printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.what());
|
||||
}
|
||||
} catch (...) { }
|
||||
}
|
||||
|
|
@ -202,8 +204,10 @@ void ignoreExceptionExceptInterrupt(Verbosity lvl)
|
|||
throw;
|
||||
} catch (const Interrupted & e) {
|
||||
throw;
|
||||
} catch (Error & e) {
|
||||
printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg);
|
||||
} catch (std::exception & e) {
|
||||
printMsg(lvl, "error (ignored): %1%", e.what());
|
||||
printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
#include "nix/util/windows-async-pipe.hh"
|
||||
#include "nix/util/windows-error.hh"
|
||||
|
||||
|
||||
#ifdef _WIN32
|
||||
# include "nix/util/windows-async-pipe.hh"
|
||||
# include "nix/util/windows-error.hh"
|
||||
|
||||
namespace nix::windows {
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
#include "nix/util/windows-error.hh"
|
||||
|
||||
#ifdef _WIN32
|
||||
#include "nix/util/windows-error.hh"
|
||||
#include <error.h>
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
|
|
|
|||
|
|
@ -420,15 +420,8 @@ static void main_nix_build(int argc, char * * argv)
|
|||
state->maybePrintStats();
|
||||
|
||||
auto buildPaths = [&](const std::vector<DerivedPath> & paths) {
|
||||
/* Note: we do this even when !printMissing to efficiently
|
||||
fetch binary cache data. */
|
||||
uint64_t downloadSize, narSize;
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
store->queryMissing(paths,
|
||||
willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
|
||||
if (settings.printMissing)
|
||||
printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
printMissing(ref<Store>(store), paths);
|
||||
|
||||
if (!dryRun)
|
||||
store->buildPaths(paths, buildMode, evalStore);
|
||||
|
|
@ -542,7 +535,7 @@ static void main_nix_build(int argc, char * * argv)
|
|||
|
||||
env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir.path().string();
|
||||
env["NIX_STORE"] = store->storeDir;
|
||||
env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
|
||||
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
|
||||
|
||||
auto parsedDrv = StructuredAttrs::tryParse(drv.env);
|
||||
DerivationOptions drvOptions;
|
||||
|
|
|
|||
|
|
@ -146,23 +146,19 @@ static void opRealise(Strings opFlags, Strings opArgs)
|
|||
for (auto & i : opArgs)
|
||||
paths.push_back(followLinksToStorePathWithOutputs(*store, i));
|
||||
|
||||
uint64_t downloadSize, narSize;
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
store->queryMissing(
|
||||
toDerivedPaths(paths),
|
||||
willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
auto missing = store->queryMissing(toDerivedPaths(paths));
|
||||
|
||||
/* Filter out unknown paths from `paths`. */
|
||||
if (ignoreUnknown) {
|
||||
std::vector<StorePathWithOutputs> paths2;
|
||||
for (auto & i : paths)
|
||||
if (!unknown.count(i.path)) paths2.push_back(i);
|
||||
if (!missing.unknown.count(i.path)) paths2.push_back(i);
|
||||
paths = std::move(paths2);
|
||||
unknown = StorePathSet();
|
||||
missing.unknown = StorePathSet();
|
||||
}
|
||||
|
||||
if (settings.printMissing)
|
||||
printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
printMissing(ref<Store>(store), missing);
|
||||
|
||||
if (dryRun) return;
|
||||
|
||||
|
|
|
|||
|
|
@ -212,6 +212,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
|
|||
lowdown. */
|
||||
static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
|
||||
{
|
||||
// Check for aliases if subcommand has exactly one element
|
||||
if (subcommand.size() == 1) {
|
||||
auto alias = toplevel.aliases.find(subcommand[0]);
|
||||
if (alias != toplevel.aliases.end()) {
|
||||
subcommand = alias->second.replacement;
|
||||
}
|
||||
}
|
||||
|
||||
auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand));
|
||||
|
||||
evalSettings.restrictEval = false;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue