nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-17 07:52:43 +01:00
Code Activity

Merge branch 'master' into debugger-merge

Browse source
This commit is contained in:
Ben Burdette 2021-04-27 12:31:01 -06:00
commit b466d51952
651 changed files with 56182 additions and 31823 deletions
Download patch file Download diff file Expand all files Collapse all files

123
src/build-remote/build-remote.cc
View file

@ -17,7 +17,7 @@
#include "store-api.hh"
#include "derivations.hh"
#include "local-store.hh"
#include "../nix/legacy.hh"
#include "legacy.hh"
using namespace nix;
using std::cin;
@ -38,13 +38,13 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) {
static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
for (auto & feature : requiredFeatures)
if (!settings.systemFeatures.get().count(feature)) return false;
if (!store.systemFeatures.get().count(feature)) return false;
return true;
}
static int _main(int argc, char * * argv)
static int main_build_remote(int argc, char * * argv)
{
{
logger = makeJSONLogger(*logger);
@ -53,6 +53,9 @@ static int _main(int argc, char * * argv)
unsetenv("DISPLAY");
unsetenv("SSH_ASKPASS");
/* If we ever use the common args framework, make sure to
remove initPlugins below and initialize settings first.
*/
if (argc != 2)
throw UsageError("called without required arguments");
@ -71,11 +74,15 @@ static int _main(int argc, char * * argv)
initPlugins();
auto store = openStore().cast<LocalStore>();
auto store = openStore();
/* It would be more appropriate to use $XDG_RUNTIME_DIR, since
that gets cleared on reboot, but it wouldn't work on macOS. */
currentLoad = store->stateDir + "/current-load";
auto currentLoadName = "/current-load";
if (auto localStore = store.dynamic_pointer_cast<LocalFSStore>())
currentLoad = std::string { localStore->stateDir } + currentLoadName;
else
currentLoad = settings.nixStateDir + currentLoadName;
std::shared_ptr<Store> sshStore;
AutoCloseFD bestSlotLock;
@ -103,10 +110,10 @@ static int _main(int argc, char * * argv)
drvPath = store->parseStorePath(readString(source));
auto requiredFeatures = readStrings<std::set<std::string>>(source);
auto canBuildLocally = amWilling
auto canBuildLocally = amWilling
&& ( neededSystem == settings.thisSystem
|| settings.extraPlatforms.get().count(neededSystem) > 0)
&& allSupportedLocally(requiredFeatures);
&& allSupportedLocally(*store, requiredFeatures);
/* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777);
@ -170,7 +177,42 @@ static int _main(int argc, char * * argv)
if (rightType && !canBuildLocally)
std::cerr << "# postpone\n";
else
{
// build the hint template.
string errorText =
"Failed to find a machine for remote build!\n"
"derivation: %s\nrequired (system, features): (%s, %s)";
errorText += "\n%s available machines:";
errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
for (unsigned int i = 0; i < machines.size(); ++i)
errorText += "\n(%s, %s, %s, %s)";
// add the template values.
string drvstr;
if (drvPath.has_value())
drvstr = drvPath->to_string();
else
drvstr = "<unknown>";
auto error = hintformat(errorText);
error
% drvstr
% neededSystem
% concatStringsSep<StringSet>(", ", requiredFeatures)
% machines.size();
for (auto & m : machines)
error
% concatStringsSep<vector<string>>(", ", m.systemTypes)
% m.maxJobs
% concatStringsSep<StringSet>(", ", m.supportedFeatures)
% concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
printMsg(canBuildLocally ? lvlChatty : lvlWarn, error);
std::cerr << "# decline\n";
}
break;
}
@ -186,26 +228,15 @@ static int _main(int argc, char * * argv)
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
Store::Params storeParams;
if (hasPrefix(bestMachine->storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
storeParams["log-fd"] = "4";
if (bestMachine->sshKey != "")
storeParams["ssh-key"] = bestMachine->sshKey;
}
sshStore = openStore(bestMachine->storeUri, storeParams);
sshStore = bestMachine->openStore();
sshStore->connect();
storeUri = bestMachine->storeUri;
} catch (std::exception & e) {
auto msg = chomp(drainFD(5, false));
logError({
.name = "Remote build",
.hint = hintfmt("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
(msg.empty() ? "" : ": " + msg))
});
printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
msg.empty() ? "" : ": " + msg);
bestMachine->enabled = false;
continue;
}
@ -220,7 +251,7 @@ connected:
std::cerr << "# accept\n" << storeUri << "\n";
auto inputs = readStrings<PathSet>(source);
auto outputs = readStrings<PathSet>(source);
auto wantedOutputs = readStrings<StringSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
@ -245,6 +276,7 @@ connected:
uploadLock = -1;
auto drv = store->readDerivation(*drvPath);
auto outputHashes = staticOutputHashes(*store, drv);
drv.inputSrcs = store->parseStorePathSet(inputs);
auto result = sshStore->buildDerivation(*drvPath, drv);
@ -252,19 +284,46 @@ connected:
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
StorePathSet missing;
for (auto & path : outputs)
if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
std::set<Realisation> missingRealisations;
StorePathSet missingPaths;
if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
for (auto & outputName : wantedOutputs) {
auto thisOutputHash = outputHashes.at(outputName);
auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
if (!store->queryRealisation(thisOutputId)) {
debug("missing output %s", outputName);
assert(result.builtOutputs.count(thisOutputId));
auto newRealisation = result.builtOutputs.at(thisOutputId);
missingRealisations.insert(newRealisation);
missingPaths.insert(newRealisation.outPath);
}
}
} else {
auto outputPaths = drv.outputsAndOptPaths(*store);
for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
assert(hopefullyOutputPath.second);
if (!store->isValidPath(*hopefullyOutputPath.second))
missingPaths.insert(*hopefullyOutputPath.second);
}
}
if (!missing.empty()) {
if (!missingPaths.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
for (auto & i : missing)
store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
for (auto & path : missingPaths)
localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
}
// XXX: Should be done as part of `copyPaths`
for (auto & realisation : missingRealisations) {
// Should hold, because if the feature isn't enabled the set
// of missing realisations should be empty
settings.requireExperimentalFeature("ca-derivations");
store->registerDrvOutput(realisation);
}
return 0;
}
}
static RegisterLegacyCommand s1("build-remote", _main);
static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote);

122
src/nix/command.cc → src/libcmd/command.cc
View file

@ -1,14 +1,37 @@
#include "command.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "derivations.hh"
#include "nixexpr.hh"
#include "profiles.hh"
#include <nlohmann/json.hpp>
extern char * * environ __attribute__((weak));
namespace nix {
Commands * RegisterCommand::commands = nullptr;
RegisterCommand::Commands * RegisterCommand::commands = nullptr;
nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix)
{
nix::Commands res;
for (auto & [name, command] : *RegisterCommand::commands)
if (name.size() == prefix.size() + 1) {
bool equal = true;
for (size_t i = 0; i < prefix.size(); ++i)
if (name[i] != prefix[i]) equal = false;
if (equal)
res.insert_or_assign(name[prefix.size()], command);
}
return res;
}
nlohmann::json NixMultiCommand::toJSON()
{
// FIXME: use Command::toJSON() as well.
return MultiCommand::toJSON();
}
StoreCommand::StoreCommand()
{
@ -55,57 +78,72 @@ ref<EvalState> EvalCommand::getEvalState()
return ref<EvalState>(evalState);
}
StorePathsCommand::StorePathsCommand(bool recursive)
RealisedPathsCommand::RealisedPathsCommand(bool recursive)
: recursive(recursive)
{
if (recursive)
addFlag({
.longName = "no-recursive",
.description = "apply operation to specified paths only",
.description = "Apply operation to specified paths only.",
.category = installablesCategory,
.handler = {&this->recursive, false},
});
else
addFlag({
.longName = "recursive",
.shortName = 'r',
.description = "apply operation to closure of the specified paths",
.description = "Apply operation to closure of the specified paths.",
.category = installablesCategory,
.handler = {&this->recursive, true},
});
mkFlag(0, "all", "apply operation to the entire store", &all);
addFlag({
.longName = "all",
.description = "Apply the operation to every store path.",
.category = installablesCategory,
.handler = {&all, true},
});
}
void StorePathsCommand::run(ref<Store> store)
void RealisedPathsCommand::run(ref<Store> store)
{
StorePaths storePaths;
std::vector<RealisedPath> paths;
if (all) {
if (installables.size())
throw UsageError("'--all' does not expect arguments");
// XXX: Only uses opaque paths, ignores all the realisations
for (auto & p : store->queryAllValidPaths())
storePaths.push_back(p);
}
else {
for (auto & p : toStorePaths(store, realiseMode, operateOn, installables))
storePaths.push_back(p);
paths.push_back(p);
} else {
auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
if (recursive) {
StorePathSet closure;
store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false);
storePaths.clear();
for (auto & p : closure)
storePaths.push_back(p);
auto roots = std::move(pathSet);
pathSet = {};
RealisedPath::closure(*store, roots, pathSet);
}
for (auto & path : pathSet)
paths.push_back(path);
}
run(store, std::move(paths));
}
StorePathsCommand::StorePathsCommand(bool recursive)
: RealisedPathsCommand(recursive)
{
}
void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
{
StorePaths storePaths;
for (auto & p : paths)
storePaths.push_back(p.path());
run(store, std::move(storePaths));
}
void StorePathCommand::run(ref<Store> store)
void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
{
auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables);
if (storePaths.size() != 1)
throw UsageError("this command requires exactly one store path");
@ -129,7 +167,7 @@ MixProfile::MixProfile()
{
addFlag({
.longName = "profile",
.description = "profile to update",
.description = "The profile to update.",
.labels = {"path"},
.handler = {&profile},
.completer = completePath
@ -145,27 +183,35 @@ void MixProfile::updateProfile(const StorePath & storePath)
switchLink(profile2,
createGeneration(
ref<LocalFSStore>(store),
profile2, store->printStorePath(storePath)));
profile2, storePath));
}
void MixProfile::updateProfile(const Buildables & buildables)
void MixProfile::updateProfile(const DerivedPathsWithHints & buildables)
{
if (!profile) return;
std::optional<StorePath> result;
std::vector<StorePath> result;
for (auto & buildable : buildables) {
for (auto & output : buildable.outputs) {
if (result)
throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
result = output.second;
}
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
result.push_back(bo.path);
},
[&](DerivedPathWithHints::Built bfd) {
for (auto & output : bfd.outputs) {
/* Output path should be known because we just tried to
build it. */
assert(output.second);
result.push_back(*output.second);
}
},
}, buildable.raw());
}
if (!result)
throw Error("'--profile' requires that the arguments produce a single store path, but there are none");
if (result.size() != 1)
throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
updateProfile(*result);
updateProfile(result[0]);
}
MixDefaultProfile::MixDefaultProfile()
@ -178,14 +224,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
addFlag({
.longName = "ignore-environment",
.shortName = 'i',
.description = "clear the entire environment (except those specified with --keep)",
.description = "Clear the entire environment (except those specified with `--keep`).",
.handler = {&ignoreEnvironment, true},
});
addFlag({
.longName = "keep",
.shortName = 'k',
.description = "keep specified environment variable",
.description = "Keep the environment variable *name*.",
.labels = {"name"},
.handler = {[&](std::string s) { keep.insert(s); }},
});
@ -193,7 +239,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
addFlag({
.longName = "unset",
.shortName = 'u',
.description = "unset specified environment variable",
.description = "Unset the environment variable *name*.",
.labels = {"name"},
.handler = {[&](std::string s) { unset.insert(s); }},
});

61
src/nix/command.hh → src/libcmd/command.hh
View file

@ -13,6 +13,8 @@ namespace nix {
extern std::string programPath;
extern char * * savedArgv;
class EvalState;
struct Pos;
class Store;
@ -21,6 +23,13 @@ static constexpr Command::Category catSecondary = 100;
static constexpr Command::Category catUtility = 101;
static constexpr Command::Category catNixInstallation = 102;
static constexpr auto installablesCategory = "Options that change the interpretation of installables";
struct NixMultiCommand : virtual MultiCommand, virtual Command
{
nlohmann::json toJSON() override;
};
/* A command that requires a Nix store. */
struct StoreCommand : virtual Command
{
@ -43,6 +52,8 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs
EvalCommand();
std::shared_ptr<EvalState> evalState;
~EvalCommand();
};
struct MixFlakeOptions : virtual Args, EvalCommand
@ -136,7 +147,7 @@ private:
};
/* A command that operates on zero or more store paths. */
struct StorePathsCommand : public InstallablesCommand
struct RealisedPathsCommand : public InstallablesCommand
{
private:
@ -149,47 +160,67 @@ protected:
public:
StorePathsCommand(bool recursive = false);
RealisedPathsCommand(bool recursive = false);
using StoreCommand::run;
virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
void run(ref<Store> store) override;
bool useDefaultInstallables() override { return !all; }
};
/* A command that operates on exactly one store path. */
struct StorePathCommand : public InstallablesCommand
struct StorePathsCommand : public RealisedPathsCommand
{
using StoreCommand::run;
StorePathsCommand(bool recursive = false);
using RealisedPathsCommand::run;
virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
void run(ref<Store> store, std::vector<RealisedPath> paths) override;
};
/* A command that operates on exactly one store path. */
struct StorePathCommand : public StorePathsCommand
{
using StorePathsCommand::run;
virtual void run(ref<Store> store, const StorePath & storePath) = 0;
void run(ref<Store> store) override;
void run(ref<Store> store, std::vector<StorePath> storePaths) override;
};
/* A helper class for registering commands globally. */
struct RegisterCommand
{
typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
static Commands * commands;
RegisterCommand(const std::string & name,
RegisterCommand(std::vector<std::string> && name,
std::function<ref<Command>()> command)
{
if (!commands) commands = new Commands;
commands->emplace(name, command);
}
static nix::Commands getCommandsFor(const std::vector<std::string> & prefix);
};
template<class T>
static RegisterCommand registerCommand(const std::string & name)
{
return RegisterCommand(name, [](){ return make_ref<T>(); });
return RegisterCommand({name}, [](){ return make_ref<T>(); });
}
Buildables build(ref<Store> store, Realise mode,
template<class T>
static RegisterCommand registerCommand2(std::vector<std::string> && name)
{
return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
}
DerivedPathsWithHints build(ref<Store> store, Realise mode,
std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
std::set<StorePath> toStorePaths(ref<Store> store,
@ -204,6 +235,12 @@ std::set<StorePath> toDerivations(ref<Store> store,
std::vector<std::shared_ptr<Installable>> installables,
bool useDeriver = false);
std::set<RealisedPath> toRealisedPaths(
ref<Store> store,
Realise mode,
OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables);
/* Helper function to generate args that invoke $EDITOR on
filename:lineno. */
Strings editorFor(const Pos & pos);
@ -219,7 +256,7 @@ struct MixProfile : virtual StoreCommand
/* If 'profile' is set, make it point at the store path produced
by 'buildables'. */
void updateProfile(const Buildables & buildables);
void updateProfile(const DerivedPathsWithHints & buildables);
};
struct MixDefaultProfile : MixProfile
@ -249,6 +286,8 @@ void completeFlakeRefWithFragment(
const Strings & defaultFlakeAttrPaths,
std::string_view prefix);
std::string showVersions(const std::set<std::string> & versions);
void printClosureDiff(
ref<Store> store,
const StorePath & beforePath,

298
src/nix/installables.cc → src/libcmd/installables.cc
View file

@ -16,6 +16,8 @@
#include <regex>
#include <queue>
#include <nlohmann/json.hpp>
namespace nix {
void completeFlakeInputPath(
@ -26,44 +28,52 @@ void completeFlakeInputPath(
auto flake = flake::getFlake(*evalState, flakeRef, true);
for (auto & input : flake.inputs)
if (hasPrefix(input.first, prefix))
completions->insert(input.first);
completions->add(input.first);
}
MixFlakeOptions::MixFlakeOptions()
{
auto category = "Common flake-related options";
addFlag({
.longName = "recreate-lock-file",
.description = "recreate lock file from scratch",
.description = "Recreate the flake's lock file from scratch.",
.category = category,
.handler = {&lockFlags.recreateLockFile, true}
});
addFlag({
.longName = "no-update-lock-file",
.description = "do not allow any updates to the lock file",
.description = "Do not allow any updates to the flake's lock file.",
.category = category,
.handler = {&lockFlags.updateLockFile, false}
});
addFlag({
.longName = "no-write-lock-file",
.description = "do not write the newly generated lock file",
.description = "Do not write the flake's newly generated lock file.",
.category = category,
.handler = {&lockFlags.writeLockFile, false}
});
addFlag({
.longName = "no-registries",
.description = "don't use flake registries",
.description = "Don't allow lookups in the flake registries.",
.category = category,
.handler = {&lockFlags.useRegistries, false}
});
addFlag({
.longName = "commit-lock-file",
.description = "commit changes to the lock file",
.description = "Commit changes to the flake's lock file.",
.category = category,
.handler = {&lockFlags.commitLockFile, true}
});
addFlag({
.longName = "update-input",
.description = "update a specific flake input",
.description = "Update a specific flake input (ignoring its previous entry in the lock file).",
.category = category,
.labels = {"input-path"},
.handler = {[&](std::string s) {
lockFlags.inputUpdates.insert(flake::parseInputPath(s));
@ -76,9 +86,11 @@ MixFlakeOptions::MixFlakeOptions()
addFlag({
.longName = "override-input",
.description = "override a specific flake input (e.g. 'dwarffs/nixpkgs')",
.description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
.category = category,
.labels = {"input-path", "flake-url"},
.handler = {[&](std::string inputPath, std::string flakeRef) {
lockFlags.writeLockFile = false;
lockFlags.inputOverrides.insert_or_assign(
flake::parseInputPath(inputPath),
parseFlakeRef(flakeRef, absPath(".")));
@ -87,7 +99,8 @@ MixFlakeOptions::MixFlakeOptions()
addFlag({
.longName = "inputs-from",
.description = "use the inputs of the specified flake as registry entries",
.description = "Use the inputs of the specified flake as registry entries.",
.category = category,
.labels = {"flake-url"},
.handler = {[&](std::string flakeRef) {
auto evalState = getEvalState();
@ -116,22 +129,25 @@ SourceExprCommand::SourceExprCommand()
addFlag({
.longName = "file",
.shortName = 'f',
.description = "evaluate FILE rather than the default",
.description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.",
.category = installablesCategory,
.labels = {"file"},
.handler = {&file},
.completer = completePath
});
addFlag({
.longName ="expr",
.description = "evaluate attributes from EXPR",
.longName = "expr",
.description = "Interpret installables as attribute paths relative to the Nix expression *expr*.",
.category = installablesCategory,
.labels = {"expr"},
.handler = {&expr}
});
addFlag({
.longName ="derivation",
.description = "operate on the store derivation rather than its outputs",
.longName = "derivation",
.description = "Operate on the store derivation rather than its outputs.",
.category = installablesCategory,
.handler = {&operateOn, OperateOn::Derivation},
});
}
@ -183,8 +199,7 @@ void completeFlakeRefWithFragment(
auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
auto evalCache = openEvalCache(*evalState,
std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)),
true);
std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
auto root = evalCache->getRoot();
@ -212,7 +227,7 @@ void completeFlakeRefWithFragment(
auto attrPath2 = attr->getAttrPath(attr2);
/* Strip the attrpath prefix. */
attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2));
completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
}
}
}
@ -223,7 +238,7 @@ void completeFlakeRefWithFragment(
for (auto & attrPath : defaultFlakeAttrPaths) {
auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
if (!attr) continue;
completions->insert(flakeRefS + "#");
completions->add(flakeRefS + "#");
}
}
}
@ -234,10 +249,25 @@ void completeFlakeRefWithFragment(
completeFlakeRef(evalState->store, prefix);
}
/*
ref<EvalState> EvalCommand::getEvalState()
{
if (!evalState)
evalState = std::make_shared<EvalState>(searchPath, getStore());
return ref<EvalState>(evalState);
}
EvalCommand::~EvalCommand()
{
if (evalState)
evalState->printStats();
}
*/
void completeFlakeRef(ref<Store> store, std::string_view prefix)
{
if (prefix == "")
completions->insert(".");
completions->add(".");
completeDir(0, prefix);
@ -248,25 +278,25 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
std::string from2(from, 6);
if (hasPrefix(from2, prefix))
completions->insert(from2);
completions->add(from2);
} else {
if (hasPrefix(from, prefix))
completions->insert(from);
completions->add(from);
}
}
}
}
Buildable Installable::toBuildable()
DerivedPathWithHints Installable::toDerivedPathWithHints()
{
auto buildables = toBuildables();
auto buildables = toDerivedPathsWithHints();
if (buildables.size() != 1)
throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size());
return std::move(buildables[0]);
}
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
Installable::getCursors(EvalState & state, bool useEvalCache)
Installable::getCursors(EvalState & state)
{
auto evalCache =
std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
@ -275,9 +305,9 @@ Installable::getCursors(EvalState & state, bool useEvalCache)
}
std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
Installable::getCursor(EvalState & state, bool useEvalCache)
Installable::getCursor(EvalState & state)
{
auto cursors = getCursors(state, useEvalCache);
auto cursors = getCursors(state);
if (cursors.empty())
throw Error("cannot find flake attribute '%s'", what());
return cursors[0];
@ -293,24 +323,23 @@ struct InstallableStorePath : Installable
std::string what() override { return store->printStorePath(storePath); }
Buildables toBuildables() override
DerivedPathsWithHints toDerivedPathsWithHints() override
{
if (storePath.isDerivation()) {
std::map<std::string, StorePath> outputs;
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = store->readDerivation(storePath);
for (auto & [name, output] : drv.outputs)
outputs.emplace(name, output.path(*store, drv.name));
for (auto & [name, output] : drv.outputsAndOptPaths(*store))
outputs.emplace(name, output.second);
return {
Buildable {
DerivedPathWithHints::Built {
.drvPath = storePath,
.outputs = std::move(outputs)
}
};
} else {
return {
Buildable {
.drvPath = {},
.outputs = {{"out", storePath}}
DerivedPathWithHints::Opaque {
.path = storePath,
}
};
}
@ -322,37 +351,24 @@ struct InstallableStorePath : Installable
}
};
Buildables InstallableValue::toBuildables()
DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints()
{
Buildables res;
DerivedPathsWithHints res;
StorePathSet drvPaths;
std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
Buildable b{.drvPath = drv.drvPath};
drvPaths.insert(drv.drvPath);
auto outputName = drv.outputName;
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath));
b.outputs.emplace(outputName, drv.outPath);
res.push_back(std::move(b));
throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath);
}
// Hack to recognize .all: if all drvs have the same drvPath,
// merge the buildables.
if (drvPaths.size() == 1) {
Buildable b{.drvPath = *drvPaths.begin()};
for (auto & b2 : res)
for (auto & output : b2.outputs)
b.outputs.insert_or_assign(output.first, output.second);
Buildables bs;
bs.push_back(std::move(b));
return bs;
} else
return res;
for (auto & i : drvsToOutputs)
res.push_back(DerivedPathWithHints::Built { i.first, i.second });
return res;
}
struct InstallableAttrPath : InstallableValue
@ -390,7 +406,7 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
for (auto & drvInfo : drvInfos) {
res.push_back({
state->store->parseStorePath(drvInfo.queryDrvPath()),
state->store->parseStorePath(drvInfo.queryOutPath()),
state->store->maybeParseStorePath(drvInfo.queryOutPath()),
drvInfo.queryOutputName()
});
}
@ -427,12 +443,11 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
std::shared_ptr<flake::LockedFlake> lockedFlake,
bool useEvalCache)
std::shared_ptr<flake::LockedFlake> lockedFlake)
{
auto fingerprint = lockedFlake->getFingerprint();
return make_ref<nix::eval_cache::EvalCache>(
useEvalCache && evalSettings.pureEval
evalSettings.useEvalCache && evalSettings.pureEval
? std::optional { std::cref(fingerprint) }
: std::nullopt,
state,
@ -465,16 +480,36 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
return s;
}
InstallableFlake::InstallableFlake(
SourceExprCommand * cmd,
ref<EvalState> state,
FlakeRef && flakeRef,
Strings && attrPaths,
Strings && prefixes,
const flake::LockFlags & lockFlags)
: InstallableValue(state),
flakeRef(flakeRef),
attrPaths(attrPaths),
prefixes(prefixes),
lockFlags(lockFlags)
{
if (cmd && cmd->getAutoArgs(*state)->size())
throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
}
std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
{
auto lockedFlake = getLockedFlake();
auto cache = openEvalCache(*state, lockedFlake, true);
auto cache = openEvalCache(*state, lockedFlake);
auto root = cache->getRoot();
for (auto & attrPath : getActualAttrPaths()) {
auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
auto attr = root->findAlongAttrPath(
parseAttrPath(*state, attrPath),
true
);
if (!attr) continue;
if (!attr->isDerivation())
@ -484,7 +519,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto drvInfo = DerivationInfo{
std::move(drvPath),
state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()),
state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
attr->getAttr(state->sOutputName)->getString()
};
@ -524,11 +559,10 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
}
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
InstallableFlake::getCursors(EvalState & state, bool useEvalCache)
InstallableFlake::getCursors(EvalState & state)
{
auto evalCache = openEvalCache(state,
std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)),
useEvalCache);
std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
auto root = evalCache->getRoot();
@ -544,8 +578,11 @@ InstallableFlake::getCursors(EvalState & state, bool useEvalCache)
std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
{
if (!_lockedFlake)
if (!_lockedFlake) {
_lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
_lockedFlake->flake.config.apply();
// FIXME: send new config to the daemon.
}
return _lockedFlake;
}
@ -596,9 +633,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
result.push_back(std::make_shared<InstallableFlake>(
getEvalState(), std::move(flakeRef),
this,
getEvalState(),
std::move(flakeRef),
fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
getDefaultFlakeAttrPathPrefixes(), lockFlags));
getDefaultFlakeAttrPathPrefixes(),
lockFlags));
continue;
} catch (...) {
ex = std::current_exception();
@ -637,26 +677,30 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
return installables.front();
}
Buildables build(ref<Store> store, Realise mode,
DerivedPathsWithHints build(ref<Store> store, Realise mode,
std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
Buildables buildables;
DerivedPathsWithHints buildables;
std::vector<StorePathWithOutputs> pathsToBuild;
std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) {
for (auto & b : i->toBuildables()) {
if (b.drvPath) {
StringSet outputNames;
for (auto & output : b.outputs)
outputNames.insert(output.first);
pathsToBuild.push_back({*b.drvPath, outputNames});
} else
for (auto & output : b.outputs)
pathsToBuild.push_back({output.second});
for (auto & b : i->toDerivedPathsWithHints()) {
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
pathsToBuild.push_back(bo);
},
[&](DerivedPathWithHints::Built bfd) {
StringSet outputNames;
for (auto & output : bfd.outputs)
outputNames.insert(output.first);
pathsToBuild.push_back(
DerivedPath::Built{bfd.drvPath, outputNames});
},
}, b.raw());
buildables.push_back(std::move(b));
}
}
@ -669,26 +713,63 @@ Buildables build(ref<Store> store, Realise mode,
return buildables;
}
std::set<RealisedPath> toRealisedPaths(
ref<Store> store,
Realise mode,
OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables)
{
std::set<RealisedPath> res;
if (operateOn == OperateOn::Output) {
for (auto & b : build(store, mode, installables))
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
res.insert(bo.path);
},
[&](DerivedPathWithHints::Built bfd) {
auto drv = store->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*store, drv);
for (auto & output : bfd.outputs) {
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (!outputHashes.count(output.first))
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath),
output.first);
auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
auto realisation = store->queryRealisation(outputId);
if (!realisation)
throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
res.insert(RealisedPath{*realisation});
}
else {
// If ca-derivations isn't enabled, behave as if
// all the paths are opaque to keep the default
// behavior
assert(output.second);
res.insert(*output.second);
}
}
},
}, b.raw());
} else {
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
auto drvPaths = toDerivations(store, installables, true);
res.insert(drvPaths.begin(), drvPaths.end());
}
return res;
}
StorePathSet toStorePaths(ref<Store> store,
Realise mode, OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables)
{
StorePathSet outPaths;
if (operateOn == OperateOn::Output) {
for (auto & b : build(store, mode, installables))
for (auto & output : b.outputs)
outPaths.insert(output.second);
} else {
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
for (auto & i : installables)
for (auto & b : i->toBuildables())
if (b.drvPath)
outPaths.insert(*b.drvPath);
}
for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
outPaths.insert(path.path());
return outPaths;
}
@ -710,20 +791,21 @@ StorePathSet toDerivations(ref<Store> store,
StorePathSet drvPaths;
for (auto & i : installables)
for (auto & b : i->toBuildables()) {
if (!b.drvPath) {
if (!useDeriver)
throw Error("argument '%s' did not evaluate to a derivation", i->what());
for (auto & output : b.outputs) {
auto derivers = store->queryValidDerivers(output.second);
for (auto & b : i->toDerivedPathsWithHints())
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
if (!useDeriver)
throw Error("argument '%s' did not evaluate to a derivation", i->what());
auto derivers = store->queryValidDerivers(bo.path);
if (derivers.empty())
throw Error("'%s' does not have a known deriver", i->what());
// FIXME: use all derivers?
drvPaths.insert(*derivers.begin());
}
} else
drvPaths.insert(*b.drvPath);
}
},
[&](DerivedPathWithHints::Built bfd) {
drvPaths.insert(bfd.drvPath);
},
}, b.raw());
return drvPaths;
}

41
src/nix/installables.hh → src/libcmd/installables.hh
View file

@ -2,6 +2,8 @@
#include "util.hh"
#include "path.hh"
#include "path-with-outputs.hh"
#include "derived-path.hh"
#include "eval.hh"
#include "flake/flake.hh"
@ -14,14 +16,6 @@ struct SourceExprCommand;
namespace eval_cache { class EvalCache; class AttrCursor; }
struct Buildable
{
std::optional<StorePath> drvPath;
std::map<std::string, StorePath> outputs;
};
typedef std::vector<Buildable> Buildables;
struct App
{
std::vector<StorePathWithOutputs> context;
@ -35,9 +29,9 @@ struct Installable
virtual std::string what() = 0;
virtual Buildables toBuildables() = 0;
virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0;
Buildable toBuildable();
DerivedPathWithHints toDerivedPathWithHints();
App toApp(EvalState & state);
@ -54,14 +48,14 @@ struct Installable
}
virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursors(EvalState & state, bool useEvalCache);
getCursors(EvalState & state);
std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
getCursor(EvalState & state, bool useEvalCache);
getCursor(EvalState & state);
virtual FlakeRef nixpkgsFlakeRef() const
{
return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}));
return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
}
};
@ -74,13 +68,13 @@ struct InstallableValue : Installable
struct DerivationInfo
{
StorePath drvPath;
StorePath outPath;
std::optional<StorePath> outPath;
std::string outputName;
};
virtual std::vector<DerivationInfo> toDerivations() = 0;
Buildables toBuildables() override;
DerivedPathsWithHints toDerivedPathsWithHints() override;
};
struct InstallableFlake : InstallableValue
@ -91,11 +85,13 @@ struct InstallableFlake : InstallableValue
const flake::LockFlags & lockFlags;
mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
InstallableFlake(ref<EvalState> state, FlakeRef && flakeRef,
Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags)
: InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths),
prefixes(prefixes), lockFlags(lockFlags)
{ }
InstallableFlake(
SourceExprCommand * cmd,
ref<EvalState> state,
FlakeRef && flakeRef,
Strings && attrPaths,
Strings && prefixes,
const flake::LockFlags & lockFlags);
std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
@ -110,7 +106,7 @@ struct InstallableFlake : InstallableValue
std::pair<Value *, Pos> toValue(EvalState & state) override;
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursors(EvalState & state, bool useEvalCache) override;
getCursors(EvalState & state) override;
std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
@ -119,7 +115,6 @@ struct InstallableFlake : InstallableValue
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
std::shared_ptr<flake::LockedFlake> lockedFlake,
bool useEvalCache);
std::shared_ptr<flake::LockedFlake> lockedFlake);
}

0
src/nix/legacy.cc → src/libcmd/legacy.cc
View file

0
src/nix/legacy.hh → src/libcmd/legacy.hh
View file

15
src/libcmd/local.mk Normal file
View file

@ -0,0 +1,15 @@
libraries += libcmd
libcmd_NAME = libnixcmd
libcmd_DIR := $(d)
libcmd_SOURCES := $(wildcard $(d)/*.cc)
libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
libcmd_LDFLAGS = -llowdown
libcmd_LIBS = libstore libutil libexpr libmain libfetchers
$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644))

50
src/libcmd/markdown.cc Normal file
View file

@ -0,0 +1,50 @@
#include "markdown.hh"
#include "util.hh"
#include "finally.hh"
#include <sys/queue.h>
#include <lowdown.h>
namespace nix {
std::string renderMarkdownToTerminal(std::string_view markdown)
{
struct lowdown_opts opts {
.type = LOWDOWN_TERM,
.maxdepth = 20,
.cols = std::min(getWindowSize().second, (unsigned short) 80),
.hmargin = 0,
.vmargin = 0,
.feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
.oflags = 0,
};
auto doc = lowdown_doc_new(&opts);
if (!doc)
throw Error("cannot allocate Markdown document");
Finally freeDoc([&]() { lowdown_doc_free(doc); });
size_t maxn = 0;
auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
if (!node)
throw Error("cannot parse Markdown document");
Finally freeNode([&]() { lowdown_node_free(node); });
auto renderer = lowdown_term_new(&opts);
if (!renderer)
throw Error("cannot allocate Markdown renderer");
Finally freeRenderer([&]() { lowdown_term_free(renderer); });
auto buf = lowdown_buf_new(16384);
if (!buf)
throw Error("cannot allocate Markdown output buffer");
Finally freeBuffer([&]() { lowdown_buf_free(buf); });
int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
if (!rndr_res)
throw Error("allocation error while rendering Markdown");
return std::string(buf->data, buf->size);
}
}

7
src/libcmd/markdown.hh Normal file
View file

@ -0,0 +1,7 @@
#include "types.hh"
namespace nix {
std::string renderMarkdownToTerminal(std::string_view markdown);
}

9
src/libcmd/nix-cmd.pc.in Normal file
View file

@ -0,0 +1,9 @@
prefix=@prefix@
libdir=@libdir@
includedir=@includedir@
Name: Nix
Description: Nix Package Manager
Version: @PACKAGE_VERSION@
Libs: -L${libdir} -lnixcmd
Cflags: -I${includedir}/nix -std=c++17

16
src/libexpr/attr-path.cc
View file

@ -52,9 +52,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
for (auto & attr : tokens) {
/* Is i an index (integer) or a normal attribute name? */
enum { apAttr, apIndex } apType = apAttr;
unsigned int attrIndex;
if (string2Int(attr, attrIndex)) apType = apIndex;
auto attrIndex = string2Int<unsigned int>(attr);
/* Evaluate the expression. */
Value * vNew = state.allocValue();
@ -65,9 +63,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
/* It should evaluate to either a set or an expression,
according to what is specified in the attrPath. */
if (apType == apAttr) {
if (!attrIndex) {
if (v->type != tAttrs)
if (v->type() != nAttrs)
throw TypeError(
"the expression selected by the selection path '%1%' should be a set but is %2%",
attrPath,
@ -82,17 +80,17 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
pos = *a->pos;
}
else if (apType == apIndex) {
else {
if (!v->isList())
throw TypeError(
"the expression selected by the selection path '%1%' should be a list but is %2%",
attrPath,
showType(*v));
if (attrIndex >= v->listSize())
throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
if (*attrIndex >= v->listSize())
throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);
v = v->listElems()[attrIndex];
v = v->listElems()[*attrIndex];
pos = noPos;
}

4
src/libexpr/attr-set.cc
View file

@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
v = vEmptySet;
return;
}
clearValue(v);
v.type = tAttrs;
v.attrs = allocBindings(capacity);
v.mkAttrs(allocBindings(capacity));
nrAttrsets++;
nrAttrsInAttrsets += capacity;
}

3
src/libexpr/attr-set.hh
View file

@ -35,6 +35,7 @@ class Bindings
{
public:
typedef uint32_t size_t;
Pos *pos;
private:
size_t size_, capacity_;
@ -77,7 +78,7 @@ public:
auto a = get(name);
if (!a)
throw Error({
.hint = hintfmt("attribute '%s' missing", name),
.msg = hintfmt("attribute '%s' missing", name),
.errPos = pos
});

17
src/libexpr/common-eval-args.cc
View file

@ -12,16 +12,20 @@ namespace nix {
MixEvalArgs::MixEvalArgs()
{
auto category = "Common evaluation options";
addFlag({
.longName = "arg",
.description = "argument to be passed to Nix functions",
.description = "Pass the value *expr* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "expr"},
.handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
});
addFlag({
.longName = "argstr",
.description = "string-valued argument to be passed to Nix functions",
.description = "Pass the string *string* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "string"},
.handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
});
@ -29,14 +33,16 @@ MixEvalArgs::MixEvalArgs()
addFlag({
.longName = "include",
.shortName = 'I',
.description = "add a path to the list of locations used to look up <...> file names",
.description = "Add *path* to the list of locations used to look up `<...>` file names.",
.category = category,
.labels = {"path"},
.handler = {[&](std::string s) { searchPath.push_back(s); }}
});
addFlag({
.longName = "impure",
.description = "allow access to mutable paths and repositories",
.description = "Allow access to mutable paths and repositories.",
.category = category,
.handler = {[&]() {
evalSettings.pureEval = false;
}},
@ -44,7 +50,8 @@ MixEvalArgs::MixEvalArgs()
addFlag({
.longName = "override-flake",
.description = "override a flake registry value",
.description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.",
.category = category,
.labels = {"original-ref", "resolved-ref"},
.handler = {[&](std::string _from, std::string _to) {
auto from = parseFlakeRef(_from, absPath("."));

65
src/libexpr/eval-cache.cc
View file

@ -390,13 +390,14 @@ Value & AttrCursor::forceValue()
}
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type == tString)
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
else if (v.type == tPath)
cachedValue = {root->db->setString(getKey(), v.path), v.path};
else if (v.type == tBool)
if (v.type() == nString)
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
string_t{v.string.s, {}}};
else if (v.type() == nPath)
cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
else if (v.type() == nBool)
cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
else if (v.type == tAttrs)
else if (v.type() == nAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getKey()), misc_t()};
@ -405,7 +406,7 @@ Value & AttrCursor::forceValue()
return v;
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
{
if (root->db) {
if (!cachedValue)
@ -422,9 +423,12 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second))
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
else
else if (std::get_if<failed_t>(&attr->second)) {
if (forceErrors)
debug("reevaluating failed cached attribute '%s'");
else
throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
} else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
@ -438,7 +442,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
auto & v = forceValue();
if (v.type != tAttrs)
if (v.type() != nAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
@ -469,9 +473,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
return maybeGetAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
{
auto p = maybeGetAttr(name);
auto p = maybeGetAttr(name, forceErrors);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
@ -482,11 +486,11 @@ std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
return getAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
res = res->maybeGetAttr(attr);
res = res->maybeGetAttr(attr, force);
if (!res) return {};
}
return res;
@ -508,10 +512,10 @@ std::string AttrCursor::getString()
auto & v = forceValue();
if (v.type != tString && v.type != tPath)
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
if (v.type() != nString && v.type() != nPath)
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
return v.type == tString ? v.string.s : v.path;
return v.type() == nString ? v.string.s : v.path;
}
string_t AttrCursor::getStringWithContext()
@ -521,8 +525,17 @@ string_t AttrCursor::getStringWithContext()
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
bool valid = true;
for (auto & c : s->second) {
if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
valid = false;
break;
}
}
if (valid) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
}
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
@ -530,12 +543,12 @@ string_t AttrCursor::getStringWithContext()
auto & v = forceValue();
if (v.type == tString)
if (v.type() == nString)
return {v.string.s, v.getContext()};
else if (v.type == tPath)
else if (v.type() == nPath)
return {v.path, {}};
else
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
}
bool AttrCursor::getBool()
@ -554,7 +567,7 @@ bool AttrCursor::getBool()
auto & v = forceValue();
if (v.type != tBool)
if (v.type() != nBool)
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
return v.boolean;
@ -576,7 +589,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
auto & v = forceValue();
if (v.type != tAttrs)
if (v.type() != nAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
@ -600,7 +613,7 @@ bool AttrCursor::isDerivation()
StorePath AttrCursor::forceDerivation()
{
auto aDrvPath = getAttr(root->state.sDrvPath);
auto aDrvPath = getAttr(root->state.sDrvPath, true);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has

10
src/libexpr/eval-cache.hh
View file

@ -9,7 +9,9 @@
namespace nix::eval_cache {
class AttrDb;
MakeError(CachedEvalError, EvalError);
struct AttrDb;
class AttrCursor;
class EvalCache : public std::enable_shared_from_this<EvalCache>
@ -92,15 +94,15 @@ public:
std::string getAttrPathStr(Symbol name) const;
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
std::shared_ptr<AttrCursor> getAttr(Symbol name);
std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> getAttr(std::string_view name);
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
std::string getString();

20
src/libexpr/eval-inline.hh
View file

@ -10,7 +10,7 @@ namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
{
throw EvalError({
.hint = hintfmt(s),
.msg = hintfmt(s),
.errPos = pos
});
}
@ -24,7 +24,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
{
throw TypeError({
.hint = hintfmt(s, showType(v)),
.msg = hintfmt(s, showType(v)),
.errPos = pos
});
}
@ -32,23 +32,21 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
void EvalState::forceValue(Value & v, const Pos & pos)
{
if (v.type == tThunk) {
if (v.isThunk()) {
Env * env = v.thunk.env;
Expr * expr = v.thunk.expr;
try {
v.type = tBlackhole;
v.mkBlackhole();
//checkInterrupt();
expr->eval(*this, *env, v);
} catch (...) {
v.type = tThunk;
v.thunk.env = env;
v.thunk.expr = expr;
v.mkThunk(env, expr);
throw;
}
}
else if (v.type == tApp)
else if (v.isApp())
callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.type == tBlackhole)
else if (v.isBlackhole())
throwEvalError(pos, "infinite recursion encountered");
}
@ -56,7 +54,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
inline void EvalState::forceAttrs(Value & v)
{
forceValue(v);
if (v.type != tAttrs)
if (v.type() != nAttrs)
throwTypeError("value is %1% while a set was expected", v);
}
@ -64,7 +62,7 @@ inline void EvalState::forceAttrs(Value & v)
inline void EvalState::forceAttrs(Value & v, const Pos & pos)
{
forceValue(v, pos);
if (v.type != tAttrs)
if (v.type() != nAttrs)
throwTypeError(pos, "value is %1% while a set was expected", v);
}

409
src/libexpr/eval.cc
View file

@ -27,6 +27,10 @@
#include <gc/gc.h>
#include <gc/gc_cpp.h>
#include <boost/coroutine2/coroutine.hpp>
#include <boost/coroutine2/protected_fixedsize_stack.hpp>
#include <boost/context/stack_context.hpp>
#endif
namespace nix {
@ -64,7 +68,7 @@ RootValue allocRootValue(Value * v)
}
static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
{
checkInterrupt();
@ -73,7 +77,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
return;
}
switch (v.type) {
switch (v.internalType) {
case tInt:
str << v.integer;
break;
@ -87,6 +91,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
else str << *i;
str << "\"";
break;
@ -153,32 +158,27 @@ std::ostream & operator << (std::ostream & str, const Value & v)
const Value *getPrimOp(const Value &v) {
const Value * primOp = &v;
while (primOp->type == tPrimOpApp) {
while (primOp->isPrimOpApp()) {
primOp = primOp->primOpApp.left;
}
assert(primOp->type == tPrimOp);
assert(primOp->isPrimOp());
return primOp;
}
string showType(ValueType type)
{
switch (type) {
case tInt: return "an integer";
case tBool: return "a Boolean";
case tString: return "a string";
case tPath: return "a path";
case tNull: return "null";
case tAttrs: return "a set";
case tList1: case tList2: case tListN: return "a list";
case tThunk: return "a thunk";
case tApp: return "a function application";
case tLambda: return "a function";
case tBlackhole: return "a black hole";
case tPrimOp: return "a built-in function";
case tPrimOpApp: return "a partially applied built-in function";
case tExternal: return "an external value";
case tFloat: return "a float";
case nInt: return "an integer";
case nBool: return "a Boolean";
case nString: return "a string";
case nPath: return "a path";
case nNull: return "null";
case nAttrs: return "a set";
case nList: return "a list";
case nFunction: return "a function";
case nExternal: return "an external value";
case nFloat: return "a float";
case nThunk: return "a thunk";
}
abort();
}
@ -186,28 +186,41 @@ string showType(ValueType type)
string showType(const Value & v)
{
switch (v.type) {
switch (v.internalType) {
case tString: return v.string.context ? "a string with context" : "a string";
case tPrimOp:
return fmt("the built-in function '%s'", string(v.primOp->name));
case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType();
case tThunk: return "a thunk";
case tApp: return "a function application";
case tBlackhole: return "a black hole";
default:
return showType(v.type);
return showType(v.type());
}
}
Pos Value::determinePos(const Pos &pos) const
{
switch (internalType) {
case tAttrs: return *attrs->pos;
case tLambda: return lambda.fun->pos;
case tApp: return app.left->determinePos(pos);
default: return pos;
}
}
bool Value::isTrivial() const
{
return
type != tApp
&& type != tPrimOpApp
&& (type != tThunk
internalType != tApp
&& internalType != tPrimOpApp
&& (internalType != tThunk
|| (dynamic_cast<ExprAttrs *>(thunk.expr)
&& ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk.expr));
|| dynamic_cast<ExprLambda *>(thunk.expr)
|| dynamic_cast<ExprList *>(thunk.expr));
}
@ -218,6 +231,31 @@ static void * oomHandler(size_t requested)
/* Convert this to a proper C++ exception. */
throw std::bad_alloc();
}
class BoehmGCStackAllocator : public StackAllocator {
boost::coroutines2::protected_fixedsize_stack stack {
// We allocate 8 MB, the default max stack size on NixOS.
// A smaller stack might be quicker to allocate but reduces the stack
// depth available for source filter expressions etc.
std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
};
public:
boost::context::stack_context allocate() override {
auto sctx = stack.allocate();
GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
return sctx;
}
void deallocate(boost::context::stack_context sctx) override {
GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
stack.deallocate(sctx);
}
};
static BoehmGCStackAllocator boehmGCStackAllocator;
#endif
@ -255,6 +293,8 @@ void initGC()
GC_set_oom_fn(oomHandler);
StackAllocator::defaultAllocator = &boehmGCStackAllocator;
/* Set the initial heap size to something fairly big (25% of
physical RAM, up to a maximum of 384 MiB) so that in most cases
we don't need to garbage collect at all. (Collection has a
@ -355,6 +395,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sEpsilon(symbols.create(""))
, repair(NoRepair)
, store(store)
, regexCache(makeRegexCache())
, baseEnv(allocEnv(128))
, staticBaseEnv(false, 0)
{
@ -369,7 +410,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
for (auto & i : _searchPath) addToSearchPath(i);
for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
}
addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
if (evalSettings.restrictEval || evalSettings.pureEval) {
allowedPaths = PathSet();
@ -381,18 +421,20 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
auto path = r.second;
if (store->isInStore(r.second)) {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path));
try {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path));
} catch (InvalidPath &) {
allowedPaths->insert(r.second);
}
} else
allowedPaths->insert(r.second);
}
}
clearValue(vEmptySet);
vEmptySet.type = tAttrs;
vEmptySet.attrs = allocBindings(0);
vEmptySet.mkAttrs(allocBindings(0));
createBaseEnv();
}
@ -419,6 +461,8 @@ Path EvalState::checkSourcePath(const Path & path_)
*/
Path abspath = canonPath(path_);
if (hasPrefix(abspath, corepkgsPrefix)) return abspath;
for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) {
found = true;
@ -508,16 +552,14 @@ Value * EvalState::addPrimOp(const string & name,
the primop to a dummy value. */
if (arity == 0) {
auto vPrimOp = allocValue();
vPrimOp->type = tPrimOp;
vPrimOp->primOp = new PrimOp(primOp, 1, sym);
vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym });
Value v;
mkApp(v, *vPrimOp, *vPrimOp);
return addConstant(name, v);
}
Value * v = allocValue();
v->type = tPrimOp;
v->primOp = new PrimOp(primOp, arity, sym);
v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym });
staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@ -525,12 +567,55 @@ Value * EvalState::addPrimOp(const string & name,
}
Value * EvalState::addPrimOp(PrimOp && primOp)
{
/* Hack to make constants lazy: turn them into a application of
the primop to a dummy value. */
if (primOp.arity == 0) {
primOp.arity = 1;
auto vPrimOp = allocValue();
vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
Value v;
mkApp(v, *vPrimOp, *vPrimOp);
return addConstant(primOp.name, v);
}
Symbol envName = primOp.name;
if (hasPrefix(primOp.name, "__"))
primOp.name = symbols.create(std::string(primOp.name, 2));
Value * v = allocValue();
v->mkPrimOp(new PrimOp(std::move(primOp)));
staticBaseEnv.vars[envName] = baseEnvDispl;
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
return v;
}
Value & EvalState::getBuiltin(const string & name)
{
return *baseEnv.values[0]->attrs->find(symbols.create(name))->value;
}
std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
if (v.isPrimOp()) {
auto v2 = &v;
if (v2->primOp->doc)
return Doc {
.pos = noPos,
.name = v2->primOp->name,
.arity = v2->primOp->arity,
.args = v2->primOp->args,
.doc = v2->primOp->doc,
};
}
return {};
}
/* Every "format" object (even temporary) takes up a few hundred bytes
of stack space, which is a real killer in the recursive
evaluator. So here are some helper functions for throwing
@ -544,7 +629,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
{
throw EvalError({
.hint = hintfmt(s, s2),
.msg = hintfmt(s, s2),
.errPos = pos
});
}
@ -557,7 +642,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
{
throw EvalError({
.hint = hintfmt(s, s2, s3),
.msg = hintfmt(s, s2, s3),
.errPos = pos
});
}
@ -566,7 +651,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
{
// p1 is where the error occurred; p2 is a position mentioned in the message.
throw EvalError({
.hint = hintfmt(s, sym, p2),
.msg = hintfmt(s, sym, p2),
.errPos = p1
});
}
@ -574,20 +659,15 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
{
throw TypeError({
.hint = hintfmt(s),
.msg = hintfmt(s),
.errPos = pos
});
}
LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
{
throw TypeError(s, s1);
}
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
{
throw TypeError({
.hint = hintfmt(s, fun.showNamePos(), s2),
.msg = hintfmt(s, fun.showNamePos(), s2),
.errPos = pos
});
}
@ -595,7 +675,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
{
throw AssertionError({
.hint = hintfmt(s, s1),
.msg = hintfmt(s, s1),
.errPos = pos
});
}
@ -603,7 +683,15 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
{
throw UndefinedVarError({
.hint = hintfmt(s, s1),
.msg = hintfmt(s, s1),
.errPos = pos
});
}
LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1))
{
throw MissingArgumentError({
.msg = hintfmt(s, s1),
.errPos = pos
});
}
@ -621,15 +709,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con
void mkString(Value & v, const char * s)
{
mkStringNoCopy(v, dupString(s));
v.mkString(dupString(s));
}
Value & mkString(Value & v, std::string_view s, const PathSet & context)
{
v.type = tString;
v.string.s = dupStringWithLen(s.data(), s.size());
v.string.context = 0;
v.mkString(dupStringWithLen(s.data(), s.size()));
if (!context.empty()) {
size_t n = 0;
v.string.context = (const char * *)
@ -644,7 +730,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context)
void mkPath(Value & v, const char * s)
{
mkPathNoCopy(v, dupString(s));
v.mkPath(dupString(s));
}
@ -705,16 +791,9 @@ Env & EvalState::allocEnv(size_t size)
void EvalState::mkList(Value & v, size_t size)
{
clearValue(v);
if (size == 1)
v.type = tList1;
else if (size == 2)
v.type = tList2;
else {
v.type = tListN;
v.bigList.size = size;
v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0;
}
v.mkList(size);
if (size > 2)
v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *));
nrListElems += size;
}
@ -723,9 +802,7 @@ unsigned long nrThunks = 0;
static inline void mkThunk(Value & v, Env & env, Expr * expr)
{
v.type = tThunk;
v.thunk.env = &env;
v.thunk.expr = expr;
v.mkThunk(&env, expr);
nrThunks++;
}
@ -860,7 +937,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e)
{
Value v;
e->eval(*this, env, v);
if (v.type != tBool)
if (v.type() != nBool)
throwTypeError("value is %1% while a Boolean was expected", v);
return v.boolean;
}
@ -870,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
{
Value v;
e->eval(*this, env, v);
if (v.type != tBool)
if (v.type() != nBool)
throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean;
}
@ -879,7 +956,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v)
{
e->eval(*this, env, v);
if (v.type != tAttrs)
if (v.type() != nAttrs)
throwTypeError("value is %1% while a set was expected", v);
}
@ -979,7 +1056,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
Value nameVal;
i.nameExpr->eval(state, *dynamicEnv, nameVal);
state.forceValue(nameVal, i.pos);
if (nameVal.type == tNull)
if (nameVal.type() == nNull)
continue;
state.forceStringNoCtx(nameVal);
Symbol nameSym = state.symbols.create(nameVal.string.s);
@ -992,6 +1069,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos));
v.attrs->sort(); // FIXME: inefficient
}
v.attrs->pos = &pos;
}
@ -1064,7 +1143,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
Symbol name = getName(i, state, env);
if (def) {
state.forceValue(*vAttrs, pos);
if (vAttrs->type != tAttrs ||
if (vAttrs->type() != nAttrs ||
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
{
def->eval(state, env, v);
@ -1104,7 +1183,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
state.forceValue(*vAttrs);
Bindings::iterator j;
Symbol name = getName(i, state, env);
if (vAttrs->type != tAttrs ||
if (vAttrs->type() != nAttrs ||
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
{
mkBool(v, false);
@ -1120,9 +1199,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
void ExprLambda::eval(EvalState & state, Env & env, Value & v)
{
v.type = tLambda;
v.lambda.env = &env;
v.lambda.fun = this;
v.mkLambda(&env, this);
}
@ -1140,11 +1217,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
/* Figure out the number of arguments still needed. */
size_t argsDone = 0;
Value * primOp = &fun;
while (primOp->type == tPrimOpApp) {
while (primOp->isPrimOpApp()) {
argsDone++;
primOp = primOp->primOpApp.left;
}
assert(primOp->type == tPrimOp);
assert(primOp->isPrimOp());
auto arity = primOp->primOp->arity;
auto argsLeft = arity - argsDone;
@ -1155,7 +1232,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
Value * vArgs[arity];
auto n = arity - 1;
vArgs[n--] = &arg;
for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left)
vArgs[n--] = arg->primOpApp.right;
/* And call the primop. */
@ -1165,9 +1242,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
} else {
Value * fun2 = allocValue();
*fun2 = fun;
v.type = tPrimOpApp;
v.primOpApp.left = fun2;
v.primOpApp.right = &arg;
v.mkPrimOpApp(fun2, &arg);
}
}
@ -1179,12 +1254,12 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
forceValue(fun, pos);
if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
if (fun.isPrimOp() || fun.isPrimOpApp()) {
callPrimOp(fun, arg, v, pos);
return;
}
if (fun.type == tAttrs) {
if (fun.type() == nAttrs) {
auto found = fun.attrs->find(sFunctor);
if (found != fun.attrs->end()) {
/* fun may be allocated on the stack of the calling function,
@ -1200,9 +1275,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
}
}
if (fun.type != tLambda) {
if (!fun.isLambda()) {
auto error = TypeError({
.hint = hintfmt("attempt to call something which is not a function but %1%", showType(fun)),
// .hint = hintfmt("attempt to call something which is not a function but %1%", showType(fun)),
.hint = hintfmt("attempt to call something which is not a function but %1%", fun),
.errPos = pos
});
if (debuggerHook)
@ -1290,7 +1366,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
{
forceValue(fun);
if (fun.type == tAttrs) {
if (fun.type() == nAttrs) {
auto found = fun.attrs->find(sFunctor);
if (found != fun.attrs->end()) {
Value * v = allocValue();
@ -1300,20 +1376,37 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
}
}
if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) {
if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
res = fun;
return;
}
Value * actualArgs = allocValue();
mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size());
mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
for (auto & i : fun.lambda.fun->formals->formals) {
Bindings::iterator j = args.find(i.name);
if (j != args.end())
actualArgs->attrs->push_back(*j);
else if (!i.def)
throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
if (fun.lambda.fun->formals->ellipsis) {
// If the formals have an ellipsis (eg the function accepts extra args) pass
// all available automatic arguments (which includes arguments specified on
// the command line via --arg/--argstr)
for (auto& v : args) {
actualArgs->attrs->push_back(v);
}
} else {
// Otherwise, only pass the arguments that the function accepts
for (auto & i : fun.lambda.fun->formals->formals) {
Bindings::iterator j = args.find(i.name);
if (j != args.end()) {
actualArgs->attrs->push_back(*j);
} else if (!i.def) {
throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
Nix attempted to evaluate a function as a top level expression; in
this case it must have its arguments supplied either by default
values, or passed explicitly with '--arg' or '--argstr'. See
https://nixos.org/manual/nix/stable/#ss-functions.)", i.name);
}
}
}
actualArgs->attrs->sort();
@ -1345,7 +1438,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
if (!state.evalBool(env, cond, pos)) {
std::ostringstream out;
cond->show(out);
throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
throwAssertionError(pos, "assertion '%1%' failed", out.str());
}
body->eval(state, env, v);
}
@ -1473,7 +1566,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
NixFloat nf = 0;
bool first = !forceString;
ValueType firstType = tString;
ValueType firstType = nString;
for (auto & i : *es) {
Value vTmp;
@ -1484,36 +1577,36 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
since paths are copied when they are used in a derivation),
and none of the strings are allowed to have contexts. */
if (first) {
firstType = vTmp.type;
firstType = vTmp.type();
first = false;
}
if (firstType == tInt) {
if (vTmp.type == tInt) {
if (firstType == nInt) {
if (vTmp.type() == nInt) {
n += vTmp.integer;
} else if (vTmp.type == tFloat) {
} else if (vTmp.type() == nFloat) {
// Upgrade the type from int to float;
firstType = tFloat;
firstType = nFloat;
nf = n;
nf += vTmp.fpoint;
} else
throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
} else if (firstType == tFloat) {
if (vTmp.type == tInt) {
} else if (firstType == nFloat) {
if (vTmp.type() == nInt) {
nf += vTmp.integer;
} else if (vTmp.type == tFloat) {
} else if (vTmp.type() == nFloat) {
nf += vTmp.fpoint;
} else
throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
} else
s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
s << state.coerceToString(pos, vTmp, context, false, firstType == nString);
}
if (firstType == tInt)
if (firstType == nInt)
mkInt(v, n);
else if (firstType == tFloat)
else if (firstType == nFloat)
mkFloat(v, nf);
else if (firstType == tPath) {
else if (firstType == nPath) {
if (!context.empty())
throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
auto path = canonPath(s.str());
@ -1540,7 +1633,7 @@ void EvalState::forceValueDeep(Value & v)
forceValue(v);
if (v.type == tAttrs) {
if (v.type() == nAttrs) {
for (auto & i : *v.attrs)
try {
recurse(*i.value);
@ -1563,7 +1656,7 @@ void EvalState::forceValueDeep(Value & v)
NixInt EvalState::forceInt(Value & v, const Pos & pos)
{
forceValue(v, pos);
if (v.type != tInt)
if (v.type() != nInt)
throwTypeError(pos, "value is %1% while an integer was expected", v);
return v.integer;
}
@ -1572,9 +1665,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
{
forceValue(v, pos);
if (v.type == tInt)
if (v.type() == nInt)
return v.integer;
else if (v.type != tFloat)
else if (v.type() != nFloat)
throwTypeError(pos, "value is %1% while a float was expected", v);
return v.fpoint;
}
@ -1583,7 +1676,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
bool EvalState::forceBool(Value & v, const Pos & pos)
{
forceValue(v, pos);
if (v.type != tBool)
if (v.type() != nBool)
throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean;
}
@ -1591,14 +1684,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
bool EvalState::isFunctor(Value & fun)
{
return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
}
void EvalState::forceFunction(Value & v, const Pos & pos)
{
forceValue(v, pos);
if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
if (v.type() != nFunction && !isFunctor(v))
throwTypeError(pos, "value is %1% while a function was expected", v);
}
@ -1606,7 +1699,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
string EvalState::forceString(Value & v, const Pos & pos)
{
forceValue(v, pos);
if (v.type != tString) {
if (v.type() != nString) {
if (pos)
throwTypeError(pos, "value is %1% while a string was expected", v);
else
@ -1639,7 +1732,7 @@ void copyContext(const Value & v, PathSet & context)
std::vector<std::pair<Path, std::string>> Value::getContext()
{
std::vector<std::pair<Path, std::string>> res;
assert(type == tString);
assert(internalType == tString);
if (string.context)
for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(*p));
@ -1672,11 +1765,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
bool EvalState::isDerivation(Value & v)
{
if (v.type != tAttrs) return false;
if (v.type() != nAttrs) return false;
Bindings::iterator i = v.attrs->find(sType);
if (i == v.attrs->end()) return false;
forceValue(*i->value);
if (i->value->type != tString) return false;
if (i->value->type() != nString) return false;
return strcmp(i->value->string.s, "derivation") == 0;
}
@ -1701,17 +1794,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
string s;
if (v.type == tString) {
if (v.type() == nString) {
copyContext(v, context);
return v.string.s;
}
if (v.type == tPath) {
if (v.type() == nPath) {
Path path(canonPath(v.path));
return copyToStore ? copyPathToStore(context, path) : path;
}
if (v.type == tAttrs) {
if (v.type() == nAttrs) {
auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
if (maybeString) {
return *maybeString;
@ -1721,18 +1814,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
}
if (v.type == tExternal)
if (v.type() == nExternal)
return v.external->coerceToString(pos, context, coerceMore, copyToStore);
if (coerceMore) {
/* Note that `false' is represented as an empty string for
shell scripting convenience, just like `null'. */
if (v.type == tBool && v.boolean) return "1";
if (v.type == tBool && !v.boolean) return "";
if (v.type == tInt) return std::to_string(v.integer);
if (v.type == tFloat) return std::to_string(v.fpoint);
if (v.type == tNull) return "";
if (v.type() == nBool && v.boolean) return "1";
if (v.type() == nBool && !v.boolean) return "";
if (v.type() == nInt) return std::to_string(v.integer);
if (v.type() == nFloat) return std::to_string(v.fpoint);
if (v.type() == nNull) return "";
if (v.isList()) {
string result;
@ -1795,40 +1888,38 @@ bool EvalState::eqValues(Value & v1, Value & v2)
if (&v1 == &v2) return true;
// Special case type-compatibility between float and int
if (v1.type == tInt && v2.type == tFloat)
if (v1.type() == nInt && v2.type() == nFloat)
return v1.integer == v2.fpoint;
if (v1.type == tFloat && v2.type == tInt)
if (v1.type() == nFloat && v2.type() == nInt)
return v1.fpoint == v2.integer;
// All other types are not compatible with each other.
if (v1.type != v2.type) return false;
if (v1.type() != v2.type()) return false;
switch (v1.type) {
switch (v1.type()) {
case tInt:
case nInt:
return v1.integer == v2.integer;
case tBool:
case nBool:
return v1.boolean == v2.boolean;
case tString:
case nString:
return strcmp(v1.string.s, v2.string.s) == 0;
case tPath:
case nPath:
return strcmp(v1.path, v2.path) == 0;
case tNull:
case nNull:
return true;
case tList1:
case tList2:
case tListN:
case nList:
if (v1.listSize() != v2.listSize()) return false;
for (size_t n = 0; n < v1.listSize(); ++n)
if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
return true;
case tAttrs: {
case nAttrs: {
/* If both sets denote a derivation (type = "derivation"),
then compare their outPaths. */
if (isDerivation(v1) && isDerivation(v2)) {
@ -1850,15 +1941,13 @@ bool EvalState::eqValues(Value & v1, Value & v2)
}
/* Functions are incomparable. */
case tLambda:
case tPrimOp:
case tPrimOpApp:
case nFunction:
return false;
case tExternal:
case nExternal:
return *v1.external == *v2.external;
case tFloat:
case nFloat:
return v1.fpoint == v2.fpoint;
default:
@ -1987,7 +2076,7 @@ void EvalState::printStats()
string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
{
throw TypeError({
.hint = hintfmt("cannot coerce %1% to a string", showType()),
.msg = hintfmt("cannot coerce %1% to a string", showType()),
.errPos = pos
});
}
@ -2013,16 +2102,28 @@ EvalSettings::EvalSettings()
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
add(getHome() + "/.nix-defexpr/channels");
add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
auto add = [&](const Path & p, const std::string & s = std::string()) {
if (pathExists(p)) {
if (s.empty()) {
res.push_back(p);
} else {
res.push_back(s + "=" + p);
}
}
};
if (!evalSettings.restrictEval && !evalSettings.pureEval) {
add(getHome() + "/.nix-defexpr/channels");
add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
add(settings.nixStateDir + "/profiles/per-user/root/channels");
}
return res;
}
EvalSettings evalSettings;
static GlobalConfig::Register r1(&evalSettings);
static GlobalConfig::Register rEvalSettings(&evalSettings);
}

81
src/libexpr/eval.hh
View file

@ -6,7 +6,6 @@
#include "symbol-table.hh"
#include "config.hh"
#include <regex>
#include <map>
#include <optional>
#include <unordered_map>
@ -30,8 +29,8 @@ struct PrimOp
PrimOpFun fun;
size_t arity;
Symbol name;
PrimOp(PrimOpFun fun, size_t arity, Symbol name)
: fun(fun), arity(arity), name(name) { }
std::vector<std::string> args;
const char * doc = nullptr;
};
@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath;
void initGC();
struct RegexCache;
std::shared_ptr<RegexCache> makeRegexCache();
class EvalState
{
public:
@ -120,7 +124,7 @@ private:
std::unordered_map<Path, Path> resolvedPaths;
/* Cache used by prim_match(). */
std::unordered_map<std::string, std::regex> regexCache;
std::shared_ptr<RegexCache> regexCache;
public:
@ -242,10 +246,23 @@ private:
Value * addPrimOp(const string & name,
size_t arity, PrimOpFun primOp);
Value * addPrimOp(PrimOp && primOp);
public:
Value & getBuiltin(const string & name);
struct Doc
{
Pos pos;
std::optional<Symbol> name;
size_t arity;
std::vector<std::string> args;
const char * doc;
};
std::optional<Doc> getDoc(Value & v);
private:
inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval);
@ -357,26 +374,64 @@ struct EvalSettings : Config
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path",
"List of directories to be searched for <...> file references."};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",
"List of directories to be searched for `<...>` file references."};
Setting<bool> restrictEval{this, false, "restrict-eval",
"Whether to restrict file system access to paths in $NIX_PATH, "
"and network access to the URI prefixes listed in 'allowed-uris'."};
Setting<bool> restrictEval{
this, false, "restrict-eval",
R"(
If set to `true`, the Nix evaluator will not allow access to any
files outside of the Nix search path (as set via the `NIX_PATH`
environment variable or the `-I` option), or to URIs outside of
`allowed-uri`. The default is `false`.
)"};
Setting<bool> pureEval{this, false, "pure-eval",
"Whether to restrict file system and network access to files specified by cryptographic hash."};
Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation",
"Whether the evaluator allows importing the result of a derivation."};
Setting<bool> enableImportFromDerivation{
this, true, "allow-import-from-derivation",
R"(
By default, Nix allows you to `import` from a derivation, allowing
building at evaluation time. With this option set to false, Nix will
throw an error when evaluating an expression that uses this feature,
allowing users to ensure their evaluation will not require any
builds to take place.
)"};
Setting<Strings> allowedUris{this, {}, "allowed-uris",
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
R"(
A list of URI prefixes to which access is allowed in restricted
evaluation mode. For example, when set to
`https://github.com/NixOS`, builtin functions such as `fetchGit` are
allowed to access `https://github.com/NixOS/patchelf.git`.
)"};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
R"(
If set to `true`, the Nix evaluator will trace every function call.
Nix will print a log message at the "vomit" level for every function
entrance and function exit.
function-trace entered undefined position at 1565795816999559622
function-trace exited undefined position at 1565795816999581277
function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
The `undefined position` means the function call is a builtin.
Use the `contrib/stack-collapse.py` script distributed with the Nix
source code to convert the trace logs in to a format suitable for
`flamegraph.pl`.
)"};
Setting<bool> useEvalCache{this, true, "eval-cache",
"Whether to use the flake evaluation cache."};
};
extern EvalSettings evalSettings;
static const std::string corepkgsPrefix{"/__corepkgs__/"};
}

41
src/libexpr/fetchurl.nix Normal file
View file

@ -0,0 +1,41 @@
{ system ? "" # obsolete
, url
, hash ? "" # an SRI hash
# Legacy hash specification
, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
, outputHash ?
if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
, outputHashAlgo ?
if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
, executable ? false
, unpack ? false
, name ? baseNameOf (toString url)
}:
derivation {
builder = "builtin:fetchurl";
# New-style output content requirements.
inherit outputHashAlgo outputHash;
outputHashMode = if unpack || executable then "recursive" else "flat";
inherit name url executable unpack;
system = "builtin";
# No need to double the amount of network traffic
preferLocalBuild = true;
impureEnvVars = [
# We borrow these environment variables from the caller to allow
# easy proxy configuration. This is impure, but a fixed-output
# derivation like fetchurl is allowed to do so since its result is
# by definition pure.
"http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
];
# To make "nix-prefetch-url" work.
urls = [ url ];
}

81
src/libexpr/flake/config.cc Normal file
View file

@ -0,0 +1,81 @@
#include "flake.hh"
#include <nlohmann/json.hpp>
namespace nix::flake {
// setting name -> setting value -> allow or ignore.
typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
Path trustedListPath()
{
return getDataDir() + "/nix/trusted-settings.json";
}
static TrustedList readTrustedList()
{
auto path = trustedListPath();
if (!pathExists(path)) return {};
auto json = nlohmann::json::parse(readFile(path));
return json;
}
static void writeTrustedList(const TrustedList & trustedList)
{
writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
}
void ConfigFile::apply()
{
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
for (auto & [name, value] : settings) {
auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
// FIXME: Move into libutil/config.cc.
std::string valueS;
if (auto s = std::get_if<std::string>(&value))
valueS = *s;
else if (auto n = std::get_if<int64_t>(&value))
valueS = fmt("%d", n);
else if (auto b = std::get_if<Explicit<bool>>(&value))
valueS = b->t ? "true" : "false";
else if (auto ss = std::get_if<std::vector<std::string>>(&value))
valueS = concatStringsSep(" ", *ss); // FIXME: evil
else
assert(false);
if (!whitelist.count(baseName)) {
auto trustedList = readTrustedList();
bool trusted = false;
if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
trusted = *saved;
} else {
// FIXME: filter ANSI escapes, newlines, \r, etc.
if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
trustedList[name][valueS] = false;
writeTrustedList(trustedList);
}
} else {
if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
trustedList[name][valueS] = trusted = true;
writeTrustedList(trustedList);
}
}
}
if (!trusted) {
warn("ignoring untrusted flake configuration setting '%s'", name);
continue;
}
}
globalConfig.set(name, valueS);
}
}
}

582
src/libexpr/flake/flake.cc
View file

@ -12,7 +12,7 @@ using namespace flake;
namespace flake {
typedef std::pair<Tree, FlakeRef> FetchedFlake;
typedef std::pair<fetchers::Tree, FlakeRef> FetchedFlake;
typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
static std::optional<FetchedFlake> lookupInFlakeCache(
@ -48,17 +48,17 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
resolvedRef = originalRef.resolve(state.store);
auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
flakeCache.push_back({resolvedRef, fetchedResolved.value()});
fetched.emplace(fetchedResolved.value());
flakeCache.push_back({resolvedRef, *fetchedResolved});
fetched.emplace(*fetchedResolved);
}
else {
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
}
}
flakeCache.push_back({originalRef, fetched.value()});
flakeCache.push_back({originalRef, *fetched});
}
auto [tree, lockedRef] = fetched.value();
auto [tree, lockedRef] = *fetched;
debug("got tree '%s' from '%s'",
state.store->printStorePath(tree.storePath), lockedRef);
@ -71,14 +71,20 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
return {std::move(tree), resolvedRef, lockedRef};
}
static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
{
if (value.isThunk() && value.isTrivial())
state.forceValue(value, pos);
}
static void expectType(EvalState & state, ValueType type,
Value & value, const Pos & pos)
{
if (value.type == tThunk && value.isTrivial())
state.forceValue(value, pos);
if (value.type != type)
forceTrivialValue(state, value, pos);
if (value.type() != type)
throw Error("expected %s but got %s at %s",
showType(type), showType(value.type), pos);
showType(type), showType(value.type()), pos);
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
@ -87,7 +93,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
static FlakeInput parseFlakeInput(EvalState & state,
const std::string & inputName, Value * value, const Pos & pos)
{
expectType(state, tAttrs, *value, pos);
expectType(state, nAttrs, *value, pos);
FlakeInput input;
@ -102,24 +108,32 @@ static FlakeInput parseFlakeInput(EvalState & state,
for (nix::Attr attr : *(value->attrs)) {
try {
if (attr.name == sUrl) {
expectType(state, tString, *attr.value, *attr.pos);
expectType(state, nString, *attr.value, *attr.pos);
url = attr.value->string.s;
attrs.emplace("url", *url);
} else if (attr.name == sFlake) {
expectType(state, tBool, *attr.value, *attr.pos);
expectType(state, nBool, *attr.value, *attr.pos);
input.isFlake = attr.value->boolean;
} else if (attr.name == sInputs) {
input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
} else if (attr.name == sFollows) {
expectType(state, tString, *attr.value, *attr.pos);
expectType(state, nString, *attr.value, *attr.pos);
input.follows = parseInputPath(attr.value->string.s);
} else {
state.forceValue(*attr.value);
if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s);
else
throw TypeError("flake input attribute '%s' is %s while a string is expected",
attr.name, showType(*attr.value));
switch (attr.value->type()) {
case nString:
attrs.emplace(attr.name, attr.value->string.s);
break;
case nBool:
attrs.emplace(attr.name, Explicit<bool> { attr.value->boolean });
break;
case nInt:
attrs.emplace(attr.name, (long unsigned int)attr.value->integer);
break;
default:
throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
attr.name, showType(*attr.value));
}
}
} catch (Error & e) {
e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
@ -153,7 +167,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
{
std::map<FlakeId, FlakeInput> inputs;
expectType(state, tAttrs, *value, pos);
expectType(state, nAttrs, *value, pos);
for (nix::Attr & inputAttr : *(*value).attrs) {
inputs.emplace(inputAttr.name,
@ -194,15 +208,10 @@ static Flake getFlake(
Value vInfo;
state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
if (vInfo.attrs->get(sEdition))
warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, tString, *description->value, *description->pos);
expectType(state, nString, *description->value, *description->pos);
flake.description = description->value->string.s;
}
@ -214,11 +223,10 @@ static Flake getFlake(
auto sOutputs = state.symbols.create("outputs");
if (auto outputs = vInfo.attrs->get(sOutputs)) {
expectType(state, tLambda, *outputs->value, *outputs->pos);
flake.vOutputs = allocRootValue(outputs->value);
expectType(state, nFunction, *outputs->value, *outputs->pos);
if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
for (auto & formal : outputs->value->lambda.fun->formals->formals) {
if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput {
.ref = parseFlakeRef(formal.name)
@ -229,11 +237,41 @@ static Flake getFlake(
} else
throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
auto sNixConfig = state.symbols.create("nixConfig");
if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos);
for (auto & setting : *nixConfig->value->attrs) {
forceTrivialValue(state, *setting.value, *setting.pos);
if (setting.value->type() == nString)
flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
else if (setting.value->type() == nInt)
flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
else if (setting.value->type() == nBool)
flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
else if (setting.value->type() == nList) {
std::vector<std::string> ss;
for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
auto elem = setting.value->listElems()[n];
if (elem->type() != nString)
throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
setting.name, showType(*setting.value));
ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));
}
flake.config.settings.insert({setting.name, ss});
}
else
throw TypeError("flake configuration setting '%s' is %s",
setting.name, showType(*setting.value));
}
}
for (auto & attr : *vInfo.attrs) {
if (attr.name != sEdition &&
attr.name != state.sDescription &&
if (attr.name != state.sDescription &&
attr.name != sInputs &&
attr.name != sOutputs)
attr.name != sOutputs &&
attr.name != sNixConfig)
throw Error("flake '%s' has an unsupported attribute '%s', at %s",
lockedRef, attr.name, *attr.pos);
}
@ -248,7 +286,7 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup
}
/* Compute an in-memory lock file for the specified top-level flake,
and optionally write it to file, it the flake is writable. */
and optionally write it to file, if the flake is writable. */
LockedFlake lockFlake(
EvalState & state,
const FlakeRef & topRef,
@ -260,284 +298,298 @@ LockedFlake lockFlake(
auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
// FIXME: symlink attack
auto oldLockFile = LockFile::read(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
try {
debug("old lock file: %s", oldLockFile);
// FIXME: symlink attack
auto oldLockFile = LockFile::read(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
// FIXME: check whether all overrides are used.
std::map<InputPath, FlakeInput> overrides;
std::set<InputPath> overridesUsed, updatesUsed;
debug("old lock file: %s", oldLockFile);
for (auto & i : lockFlags.inputOverrides)
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
// FIXME: check whether all overrides are used.
std::map<InputPath, FlakeInput> overrides;
std::set<InputPath> overridesUsed, updatesUsed;
LockFile newLockFile;
for (auto & i : lockFlags.inputOverrides)
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
std::vector<FlakeRef> parents;
LockFile newLockFile;
std::function<void(
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)>
computeLocks;
std::vector<FlakeRef> parents;
computeLocks = [&](
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)
{
debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
std::function<void(
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)>
computeLocks;
/* Get the overrides (i.e. attributes of the form
'inputs.nixops.inputs.nixpkgs.url = ...'). */
// FIXME: check this
for (auto & [id, input] : flake.inputs) {
for (auto & [idOverride, inputOverride] : input.overrides) {
computeLocks = [&](
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)
{
debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
/* Get the overrides (i.e. attributes of the form
'inputs.nixops.inputs.nixpkgs.url = ...'). */
// FIXME: check this
for (auto & [id, input] : flake.inputs) {
for (auto & [idOverride, inputOverride] : input.overrides) {
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
inputPath.push_back(idOverride);
overrides.insert_or_assign(inputPath, inputOverride);
}
}
/* Go over the flake inputs, resolve/fetch them if
necessary (i.e. if they're new or the flakeref changed
from what's in the lock file). */
for (auto & [id, input2] : flakeInputs) {
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
inputPath.push_back(idOverride);
overrides.insert_or_assign(inputPath, inputOverride);
}
}
auto inputPathS = printInputPath(inputPath);
debug("computing input '%s'", inputPathS);
/* Go over the flake inputs, resolve/fetch them if
necessary (i.e. if they're new or the flakeref changed
from what's in the lock file). */
for (auto & [id, input2] : flakeInputs) {
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
auto inputPathS = printInputPath(inputPath);
debug("computing input '%s'", inputPathS);
try {
/* Do we have an override for this input from one of the
ancestors? */
auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end();
if (hasOverride) overridesUsed.insert(inputPath);
auto & input = hasOverride ? i->second : input2;
/* Do we have an override for this input from one of the
ancestors? */
auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end();
if (hasOverride) overridesUsed.insert(inputPath);
auto & input = hasOverride ? i->second : input2;
/* Resolve 'follows' later (since it may refer to an input
path we haven't processed yet. */
if (input.follows) {
InputPath target;
if (hasOverride || input.absolute)
/* 'follows' from an override is relative to the
root of the graph. */
target = *input.follows;
else {
/* Otherwise, it's relative to the current flake. */
target = inputPathPrefix;
for (auto & i : *input.follows) target.push_back(i);
}
debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
node->inputs.insert_or_assign(id, target);
continue;
}
/* Resolve 'follows' later (since it may refer to an input
path we haven't processed yet. */
if (input.follows) {
InputPath target;
if (hasOverride || input.absolute)
/* 'follows' from an override is relative to the
root of the graph. */
target = *input.follows;
else {
/* Otherwise, it's relative to the current flake. */
target = inputPathPrefix;
for (auto & i : *input.follows) target.push_back(i);
}
debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
node->inputs.insert_or_assign(id, target);
continue;
}
assert(input.ref);
assert(input.ref);
/* Do we have an entry in the existing lock file? And we
don't have a --update-input flag for this input? */
std::shared_ptr<LockedNode> oldLock;
/* Do we have an entry in the existing lock file? And we
don't have a --update-input flag for this input? */
std::shared_ptr<LockedNode> oldLock;
updatesUsed.insert(inputPath);
updatesUsed.insert(inputPath);
if (oldNode && !lockFlags.inputUpdates.count(inputPath))
if (auto oldLock2 = get(oldNode->inputs, id))
if (auto oldLock3 = std::get_if<0>(&*oldLock2))
oldLock = *oldLock3;
if (oldNode && !lockFlags.inputUpdates.count(inputPath))
if (auto oldLock2 = get(oldNode->inputs, id))
if (auto oldLock3 = std::get_if<0>(&*oldLock2))
oldLock = *oldLock3;
if (oldLock
&& oldLock->originalRef == *input.ref
&& !hasOverride)
{
debug("keeping existing input '%s'", inputPathS);
if (oldLock
&& oldLock->originalRef == *input.ref
&& !hasOverride)
{
debug("keeping existing input '%s'", inputPathS);
/* Copy the input from the old lock since its flakeref
didn't change and there is no override from a
higher level flake. */
auto childNode = std::make_shared<LockedNode>(
oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
/* Copy the input from the old lock since its flakeref
didn't change and there is no override from a
higher level flake. */
auto childNode = std::make_shared<LockedNode>(
oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
node->inputs.insert_or_assign(id, childNode);
node->inputs.insert_or_assign(id, childNode);
/* If we have an --update-input flag for an input
of this input, then we must fetch the flake to
to update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
/* If we have an --update-input flag for an input
of this input, then we must fetch the flake to
update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
auto hasChildUpdate =
lb != lockFlags.inputUpdates.end()
&& lb->size() > inputPath.size()
&& std::equal(inputPath.begin(), inputPath.end(), lb->begin());
auto hasChildUpdate =
lb != lockFlags.inputUpdates.end()
&& lb->size() > inputPath.size()
&& std::equal(inputPath.begin(), inputPath.end(), lb->begin());
if (hasChildUpdate) {
auto inputFlake = getFlake(
state, oldLock->lockedRef, false, flakeCache);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
} else {
/* No need to fetch this flake, we can be
lazy. However there may be new overrides on the
inputs of this flake, so we need to check
those. */
FlakeInputs fakeInputs;
if (hasChildUpdate) {
auto inputFlake = getFlake(
state, oldLock->lockedRef, false, flakeCache);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
} else {
/* No need to fetch this flake, we can be
lazy. However there may be new overrides on the
inputs of this flake, so we need to check
those. */
FlakeInputs fakeInputs;
for (auto & i : oldLock->inputs) {
if (auto lockedNode = std::get_if<0>(&i.second)) {
fakeInputs.emplace(i.first, FlakeInput {
.ref = (*lockedNode)->originalRef,
.isFlake = (*lockedNode)->isFlake,
});
} else if (auto follows = std::get_if<1>(&i.second)) {
fakeInputs.emplace(i.first, FlakeInput {
.follows = *follows,
.absolute = true
});
for (auto & i : oldLock->inputs) {
if (auto lockedNode = std::get_if<0>(&i.second)) {
fakeInputs.emplace(i.first, FlakeInput {
.ref = (*lockedNode)->originalRef,
.isFlake = (*lockedNode)->isFlake,
});
} else if (auto follows = std::get_if<1>(&i.second)) {
fakeInputs.emplace(i.first, FlakeInput {
.follows = *follows,
.absolute = true
});
}
}
computeLocks(fakeInputs, childNode, inputPath, oldLock);
}
} else {
/* We need to create a new lock file entry. So fetch
this input. */
debug("creating new input '%s'", inputPathS);
if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
throw Error("cannot update flake input '%s' in pure mode", inputPathS);
if (input.isFlake) {
auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
/* Note: in case of an --override-input, we use
the *original* ref (input2.ref) for the
"original" field, rather than the
override. This ensures that the override isn't
nuked the next time we update the lock
file. That is, overrides are sticky unless you
use --no-write-lock-file. */
auto childNode = std::make_shared<LockedNode>(
inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
node->inputs.insert_or_assign(id, childNode);
/* Guard against circular flake imports. */
for (auto & parent : parents)
if (parent == *input.ref)
throw Error("found circular import of flake '%s'", parent);
parents.push_back(*input.ref);
Finally cleanup([&]() { parents.pop_back(); });
/* Recursively process the inputs of this
flake. Also, unless we already have this flake
in the top-level lock file, use this flake's
own lock file. */
computeLocks(
inputFlake.inputs, childNode, inputPath,
oldLock
? std::dynamic_pointer_cast<const Node>(oldLock)
: LockFile::read(
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
}
else {
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, lockFlags.useRegistries, flakeCache);
node->inputs.insert_or_assign(id,
std::make_shared<LockedNode>(lockedRef, *input.ref, false));
}
}
computeLocks(fakeInputs, childNode, inputPath, oldLock);
}
} else {
/* We need to create a new lock file entry. So fetch
this input. */
debug("creating new input '%s'", inputPathS);
if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
throw Error("cannot update flake input '%s' in pure mode", inputPathS);
if (input.isFlake) {
auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
/* Note: in case of an --override-input, we use
the *original* ref (input2.ref) for the
"original" field, rather than the
override. This ensures that the override isn't
nuked the next time we update the lock
file. That is, overrides are sticky unless you
use --no-write-lock-file. */
auto childNode = std::make_shared<LockedNode>(
inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
node->inputs.insert_or_assign(id, childNode);
/* Guard against circular flake imports. */
for (auto & parent : parents)
if (parent == *input.ref)
throw Error("found circular import of flake '%s'", parent);
parents.push_back(*input.ref);
Finally cleanup([&]() { parents.pop_back(); });
/* Recursively process the inputs of this
flake. Also, unless we already have this flake
in the top-level lock file, use this flake's
own lock file. */
computeLocks(
inputFlake.inputs, childNode, inputPath,
oldLock
? std::dynamic_pointer_cast<const Node>(oldLock)
: LockFile::read(
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
}
else {
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, lockFlags.useRegistries, flakeCache);
node->inputs.insert_or_assign(id,
std::make_shared<LockedNode>(lockedRef, *input.ref, false));
} catch (Error & e) {
e.addTrace({}, "while updating the flake input '%s'", inputPathS);
throw;
}
}
}
};
};
computeLocks(
flake.inputs, newLockFile.root, {},
lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
computeLocks(
flake.inputs, newLockFile.root, {},
lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
for (auto & i : lockFlags.inputOverrides)
if (!overridesUsed.count(i.first))
warn("the flag '--override-input %s %s' does not match any input",
printInputPath(i.first), i.second);
for (auto & i : lockFlags.inputOverrides)
if (!overridesUsed.count(i.first))
warn("the flag '--override-input %s %s' does not match any input",
printInputPath(i.first), i.second);
for (auto & i : lockFlags.inputUpdates)
if (!updatesUsed.count(i))
warn("the flag '--update-input %s' does not match any input", printInputPath(i));
for (auto & i : lockFlags.inputUpdates)
if (!updatesUsed.count(i))
warn("the flag '--update-input %s' does not match any input", printInputPath(i));
/* Check 'follows' inputs. */
newLockFile.check();
/* Check 'follows' inputs. */
newLockFile.check();
debug("new lock file: %s", newLockFile);
debug("new lock file: %s", newLockFile);
/* Check whether we need to / can write the new lock file. */
if (!(newLockFile == oldLockFile)) {
/* Check whether we need to / can write the new lock file. */
if (!(newLockFile == oldLockFile)) {
auto diff = LockFile::diff(oldLockFile, newLockFile);
auto diff = LockFile::diff(oldLockFile, newLockFile);
if (lockFlags.writeLockFile) {
if (auto sourcePath = topRef.input.getSourcePath()) {
if (!newLockFile.isImmutable()) {
if (settings.warnDirty)
warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
} else {
if (!lockFlags.updateLockFile)
throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
if (lockFlags.writeLockFile) {
if (auto sourcePath = topRef.input.getSourcePath()) {
if (!newLockFile.isImmutable()) {
if (settings.warnDirty)
warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
} else {
if (!lockFlags.updateLockFile)
throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
auto path = *sourcePath + "/" + relPath;
auto path = *sourcePath + "/" + relPath;
bool lockFileExists = pathExists(path);
bool lockFileExists = pathExists(path);
if (lockFileExists) {
auto s = chomp(diff);
if (s.empty())
warn("updating lock file '%s'", path);
else
warn("updating lock file '%s':\n%s", path, s);
} else
warn("creating lock file '%s'", path);
if (lockFileExists) {
auto s = chomp(diff);
if (s.empty())
warn("updating lock file '%s'", path);
else
warn("updating lock file '%s':\n%s", path, s);
} else
warn("creating lock file '%s'", path);
newLockFile.write(path);
newLockFile.write(path);
topRef.input.markChangedFile(
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
lockFlags.commitLockFile
? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
relPath, lockFileExists ? "Update" : "Add", diff))
: std::nullopt);
topRef.input.markChangedFile(
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
lockFlags.commitLockFile
? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
relPath, lockFileExists ? "Update" : "Add", diff))
: std::nullopt);
/* Rewriting the lockfile changed the top-level
repo, so we should re-read it. FIXME: we could
also just clear the 'rev' field... */
auto prevLockedRef = flake.lockedRef;
FlakeCache dummyCache;
flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
/* Rewriting the lockfile changed the top-level
repo, so we should re-read it. FIXME: we could
also just clear the 'rev' field... */
auto prevLockedRef = flake.lockedRef;
FlakeCache dummyCache;
flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
if (lockFlags.commitLockFile &&
flake.lockedRef.input.getRev() &&
prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
if (lockFlags.commitLockFile &&
flake.lockedRef.input.getRev() &&
prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
/* Make sure that we picked up the change,
i.e. the tree should usually be dirty
now. Corner case: we could have reverted from a
dirty to a clean tree! */
if (flake.lockedRef.input == prevLockedRef.input
&& !flake.lockedRef.input.isImmutable())
throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
}
/* Make sure that we picked up the change,
i.e. the tree should usually be dirty
now. Corner case: we could have reverted from a
dirty to a clean tree! */
if (flake.lockedRef.input == prevLockedRef.input
&& !flake.lockedRef.input.isImmutable())
throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
}
} else
throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
} else
throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
} else
warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
}
warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
}
return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
} catch (Error & e) {
e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string());
throw;
}
}
void callFlake(EvalState & state,

43
src/libexpr/flake/flake.hh
View file

@ -17,24 +17,55 @@ struct FlakeInput;
typedef std::map<FlakeId, FlakeInput> FlakeInputs;
/* FlakeInput is the 'Flake'-level parsed form of the "input" entries
* in the flake file.
*
* A FlakeInput is normally constructed by the 'parseFlakeInput'
* function which parses the input specification in the '.flake' file
* to create a 'FlakeRef' (a fetcher, the fetcher-specific
* representation of the input specification, and possibly the fetched
* local store path result) and then creating this FlakeInput to hold
* that FlakeRef, along with anything that might override that
* FlakeRef (like command-line overrides or "follows" specifications).
*
* A FlakeInput is also sometimes constructed directly from a FlakeRef
* instead of starting at the flake-file input specification
* (e.g. overrides, follows, and implicit inputs).
*
* A FlakeInput will usually have one of either "ref" or "follows"
* set. If not otherwise specified, a "ref" will be generated to a
* 'type="indirect"' flake, which is treated as simply the name of a
* flake to be resolved in the registry.
*/
struct FlakeInput
{
std::optional<FlakeRef> ref;
bool isFlake = true;
bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path
std::optional<InputPath> follows;
bool absolute = false; // whether 'follows' is relative to the flake root
FlakeInputs overrides;
};
struct ConfigFile
{
using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>;
std::map<std::string, ConfigValue> settings;
void apply();
};
/* The contents of a flake.nix file. */
struct Flake
{
FlakeRef originalRef;
FlakeRef resolvedRef;
FlakeRef lockedRef;
FlakeRef originalRef; // the original flake specification (by the user)
FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
FlakeRef lockedRef; // the specific local store result of invoking the fetcher
std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs;
RootValue vOutputs;
ConfigFile config; // 'nixConfig' attribute
~Flake();
};
@ -82,7 +113,7 @@ struct LockFlags
/* Whether to commit changes to flake.lock. */
bool commitLockFile = false;
/* Flake inputs to be overriden. */
/* Flake inputs to be overridden. */
std::map<InputPath, FlakeRef> inputOverrides;
/* Flake inputs to be updated. This means that any existing lock

10
src/libexpr/flake/flakeref.cc
View file

@ -1,6 +1,7 @@
#include "flakeref.hh"
#include "store-api.hh"
#include "url.hh"
#include "url-parts.hh"
#include "fetchers.hh"
#include "registry.hh"
@ -15,10 +16,10 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege
std::string FlakeRef::to_string() const
{
auto url = input.toURL();
std::map<std::string, std::string> extraQuery;
if (subdir != "")
url.query.insert_or_assign("dir", subdir);
return url.to_string();
extraQuery.insert_or_assign("dir", subdir);
return input.toURLString(extraQuery);
}
fetchers::Attrs FlakeRef::toAttrs() const
@ -156,7 +157,8 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
} else {
if (!hasPrefix(path, "/"))
throw BadURL("flake reference '%s' is not an absolute path", url);
path = canonPath(path);
auto query = decodeQuery(match[2]);
path = canonPath(path + "/" + get(query, "dir").value_or(""));
}
fetchers::Attrs attrs;

23
src/libexpr/flake/flakeref.hh
View file

@ -12,10 +12,33 @@ class Store;
typedef std::string FlakeId;
/* A flake reference specifies how to fetch a flake or raw source
* (e.g. from a Git repository). It is created from a URL-like syntax
* (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
* type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
* path.
*
* Each flake will have a number of FlakeRef objects: one for each
* input to the flake.
*
* The normal method of constructing a FlakeRef is by starting with an
* input description (usually the attrs or a url from the flake file),
* locating a fetcher for that input, and then capturing the Input
* object that fetcher generates (usually via
* FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
*
* The actual fetch not have been performed yet (i.e. a FlakeRef may
* be lazy), but the fetcher can be invoked at any time via the
* FlakeRef to ensure the store is populated with this input.
*/
struct FlakeRef
{
/* fetcher-specific representation of the input, sufficient to
perform the fetch operation. */
fetchers::Input input;
/* sub-path within the fetched input that represents this input */
Path subdir;
bool operator==(const FlakeRef & other) const;

31
src/libexpr/flake/lockfile.cc
View file

@ -1,5 +1,6 @@
#include "lockfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <nlohmann/json.hpp>
@ -12,12 +13,12 @@ FlakeRef getFlakeRef(
{
auto i = json.find(attr);
if (i != json.end()) {
auto attrs = jsonToAttrs(*i);
auto attrs = fetchers::jsonToAttrs(*i);
// FIXME: remove when we drop support for version 5.
if (info) {
auto j = json.find(info);
if (j != json.end()) {
for (auto k : jsonToAttrs(*j))
for (auto k : fetchers::jsonToAttrs(*j))
attrs.insert_or_assign(k.first, k.second);
}
}
@ -33,7 +34,8 @@ LockedNode::LockedNode(const nlohmann::json & json)
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
{
if (!lockedRef.input.isImmutable())
throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
throw Error("lockfile contains mutable lock '%s'",
fetchers::attrsToJSON(lockedRef.input.toAttrs()));
}
StorePath LockedNode::computeStorePath(Store & store) const
@ -76,7 +78,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
{
if (jsonNode.find("inputs") == jsonNode.end()) return;
for (auto & i : jsonNode["inputs"].items()) {
if (i.value().is_array()) {
if (i.value().is_array()) { // FIXME: remove, obsolete
InputPath path;
for (auto & j : i.value())
path.push_back(j);
@ -85,10 +87,13 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
std::string inputKey = i.value();
auto k = nodeMap.find(inputKey);
if (k == nodeMap.end()) {
auto jsonNode2 = json["nodes"][inputKey];
auto input = std::make_shared<LockedNode>(jsonNode2);
auto nodes = json["nodes"];
auto jsonNode2 = nodes.find(inputKey);
if (jsonNode2 == nodes.end())
throw Error("lock file references missing node '%s'", inputKey);
auto input = std::make_shared<LockedNode>(*jsonNode2);
k = nodeMap.insert_or_assign(inputKey, input).first;
getInputs(*input, jsonNode2);
getInputs(*input, *jsonNode2);
}
if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
node.inputs.insert_or_assign(i.key(), child);
@ -109,7 +114,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
// a bit since we don't need to worry about cycles.
}
nlohmann::json LockFile::toJson() const
nlohmann::json LockFile::toJSON() const
{
nlohmann::json nodes;
std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
@ -153,8 +158,8 @@ nlohmann::json LockFile::toJson() const
}
if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
if (!lockedNode->isFlake) n["flake"] = false;
}
@ -173,7 +178,7 @@ nlohmann::json LockFile::toJson() const
std::string LockFile::to_string() const
{
return toJson().dump(2);
return toJSON().dump(2);
}
LockFile LockFile::read(const Path & path)
@ -184,7 +189,7 @@ LockFile LockFile::read(const Path & path)
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{
stream << lockFile.toJson().dump(2);
stream << lockFile.toJSON().dump(2);
return stream;
}
@ -222,7 +227,7 @@ bool LockFile::isImmutable() const
bool LockFile::operator ==(const LockFile & other) const
{
// FIXME: slow
return toJson() == other.toJson();
return toJSON() == other.toJSON();
}
InputPath parseInputPath(std::string_view s)

6
src/libexpr/flake/lockfile.hh
View file

@ -6,13 +6,11 @@
namespace nix {
class Store;
struct StorePath;
class StorePath;
}
namespace nix::flake {
using namespace fetchers;
typedef std::vector<FlakeId> InputPath;
struct LockedNode;
@ -54,7 +52,7 @@ struct LockFile
LockFile() {};
LockFile(const nlohmann::json & json, const Path & path);
nlohmann::json toJson() const;
nlohmann::json toJSON() const;
std::string to_string() const;

55
src/libexpr/get-drvs.cc
View file

@ -2,6 +2,7 @@
#include "util.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "path-with-outputs.hh"
#include <cstring>
#include <regex>
@ -19,7 +20,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("")
{
auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
this->drvPath = store->printStorePath(drvPath);
@ -38,8 +39,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
auto & [outputName, output] = *i;
outPath = store->printStorePath(i->second.path(*store, drv.name));
auto optStorePath = output.path(*store, drv.name, outputName);
if (optStorePath)
outPath = store->printStorePath(*optStorePath);
}
@ -77,12 +81,15 @@ string DrvInfo::queryDrvPath() const
string DrvInfo::queryOutPath() const
{
if (outPath == "" && attrs) {
if (!outPath && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
if (i != attrs->end())
outPath = state->coerceToPath(*i->pos, *i->value, context);
}
return outPath;
if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
return *outPath;
}
@ -122,7 +129,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
if (!outTI->isList()) throw errMsg;
Outputs result;
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
if ((*i)->type != tString) throw errMsg;
if ((*i)->type() != nString) throw errMsg;
auto out = outputs.find((*i)->string.s);
if (out == outputs.end()) throw errMsg;
result.insert(*out);
@ -166,20 +173,20 @@ StringSet DrvInfo::queryMetaNames()
bool DrvInfo::checkMeta(Value & v)
{
state->forceValue(v);
if (v.isList()) {
if (v.type() == nList) {
for (unsigned int n = 0; n < v.listSize(); ++n)
if (!checkMeta(*v.listElems()[n])) return false;
return true;
}
else if (v.type == tAttrs) {
else if (v.type() == nAttrs) {
Bindings::iterator i = v.attrs->find(state->sOutPath);
if (i != v.attrs->end()) return false;
for (auto & i : *v.attrs)
if (!checkMeta(*i.value)) return false;
return true;
}
else return v.type == tInt || v.type == tBool || v.type == tString ||
v.type == tFloat;
else return v.type() == nInt || v.type() == nBool || v.type() == nString ||
v.type() == nFloat;
}
@ -195,7 +202,7 @@ Value * DrvInfo::queryMeta(const string & name)
string DrvInfo::queryMetaString(const string & name)
{
Value * v = queryMeta(name);
if (!v || v->type != tString) return "";
if (!v || v->type() != nString) return "";
return v->string.s;
}
@ -204,12 +211,12 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
{
Value * v = queryMeta(name);
if (!v) return def;
if (v->type == tInt) return v->integer;
if (v->type == tString) {
if (v->type() == nInt) return v->integer;
if (v->type() == nString) {
/* Backwards compatibility with before we had support for
integer meta fields. */
NixInt n;
if (string2Int(v->string.s, n)) return n;
if (auto n = string2Int<NixInt>(v->string.s))
return *n;
}
return def;
}
@ -218,12 +225,12 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
{
Value * v = queryMeta(name);
if (!v) return def;
if (v->type == tFloat) return v->fpoint;
if (v->type == tString) {
if (v->type() == nFloat) return v->fpoint;
if (v->type() == nString) {
/* Backwards compatibility with before we had support for
float meta fields. */
NixFloat n;
if (string2Float(v->string.s, n)) return n;
if (auto n = string2Float<NixFloat>(v->string.s))
return *n;
}
return def;
}
@ -233,8 +240,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
{
Value * v = queryMeta(name);
if (!v) return def;
if (v->type == tBool) return v->boolean;
if (v->type == tString) {
if (v->type() == nBool) return v->boolean;
if (v->type() == nString) {
/* Backwards compatibility with before we had support for
Boolean meta fields. */
if (strcmp(v->string.s, "true") == 0) return true;
@ -325,7 +332,7 @@ static void getDerivations(EvalState & state, Value & vIn,
/* Process the expression. */
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
else if (v.type == tAttrs) {
else if (v.type() == nAttrs) {
/* !!! undocumented hackery to support combining channels in
nix-env.cc. */
@ -347,7 +354,7 @@ static void getDerivations(EvalState & state, Value & vIn,
/* If the value of this attribute is itself a set,
should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */
if (i->value->type == tAttrs) {
if (i->value->type() == nAttrs) {
Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
@ -356,7 +363,7 @@ static void getDerivations(EvalState & state, Value & vIn,
}
}
else if (v.isList()) {
else if (v.type() == nList) {
for (unsigned int n = 0; n < v.listSize(); ++n) {
string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))

2
src/libexpr/get-drvs.hh
View file

@ -20,7 +20,7 @@ private:
mutable string name;
mutable string system;
mutable string drvPath;
mutable string outPath;
mutable std::optional<string> outPath;
mutable string outputName;
Outputs outputs;

8
src/libexpr/json-to-value.cc
View file

@ -115,6 +115,14 @@ public:
{
return handle_value<void(Value&, const char*)>(mkString, val.c_str());
}
#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
bool binary(binary_t&)
{
// This function ought to be unreachable
assert(false);
return true;
}
#endif
bool start_object(std::size_t len)
{

4
src/libexpr/lexer.l
View file

@ -12,6 +12,10 @@
%{
#ifdef __clang__
#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
#endif
#include <boost/lexical_cast.hpp>
#include "nixexpr.hh"

6
src/libexpr/local.mk
View file

@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
libexpr_LIBS = libutil libstore libfetchers
libexpr_LDFLAGS =
libexpr_LDFLAGS = -lboost_context
ifneq ($(OS), FreeBSD)
libexpr_LDFLAGS += -ldl
endif
@ -35,13 +35,11 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh

2
src/libexpr/nixexpr.cc
View file

@ -284,7 +284,7 @@ void ExprVar::bindVars(const StaticEnv & env)
"undefined variable" error now. */
if (withLevel == -1)
throw UndefinedVarError({
.hint = hintfmt("undefined variable '%1%'", name),
.msg = hintfmt("undefined variable '%1%'", name),
.errPos = pos
});
fromWith = true;

9
src/libexpr/nixexpr.hh
View file

@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError);
MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error);
@ -129,7 +130,7 @@ struct ExprPath : Expr
{
string s;
Value v;
ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); };
COMMON_METHODS
Value * maybeThunk(EvalState & state, Env & env);
};
@ -179,6 +180,7 @@ struct ExprOpHasAttr : Expr
struct ExprAttrs : Expr
{
bool recursive;
Pos pos;
struct AttrDef {
bool inherited;
Expr * e;
@ -198,7 +200,8 @@ struct ExprAttrs : Expr
};
typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
DynamicAttrDefs dynamicAttrs;
ExprAttrs() : recursive(false) { };
ExprAttrs(const Pos &pos) : recursive(false), pos(pos) { };
ExprAttrs() : recursive(false), pos(noPos) { };
COMMON_METHODS
};
@ -238,7 +241,7 @@ struct ExprLambda : Expr
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'", arg),
.msg = hintfmt("duplicate formal function argument '%1%'", arg),
.errPos = pos
});
};

39
src/libexpr/parser.y
View file

@ -32,7 +32,7 @@ namespace nix {
Path basePath;
Symbol file;
FileOrigin origin;
ErrorInfo error;
std::optional<ErrorInfo> error;
Symbol sLetBody;
ParseData(EvalState & state)
: state(state)
@ -66,8 +66,8 @@ namespace nix {
static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
{
throw ParseError({
.hint = hintfmt("attribute '%1%' already defined at %2%",
showAttrPath(attrPath), prevPos),
.msg = hintfmt("attribute '%1%' already defined at %2%",
showAttrPath(attrPath), prevPos),
.errPos = pos
});
}
@ -75,7 +75,7 @@ static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prev
static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
{
throw ParseError({
.hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
.msg = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
.errPos = pos
});
}
@ -146,7 +146,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
{
if (!formals->argNames.insert(formal.name).second)
throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'",
.msg = hintfmt("duplicate formal function argument '%1%'",
formal.name),
.errPos = pos
});
@ -258,7 +258,7 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
{
data->error = {
.hint = hintfmt(error),
.msg = hintfmt(error),
.errPos = makeCurPos(*loc, data)
};
}
@ -338,7 +338,7 @@ expr_function
| LET binds IN expr_function
{ if (!$2->dynamicAttrs.empty())
throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in let"),
.msg = hintfmt("dynamic attributes not allowed in let"),
.errPos = CUR_POS
});
$$ = new ExprLet($2, $4);
@ -418,7 +418,7 @@ expr_simple
static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
if (noURLLiterals)
throw ParseError({
.hint = hintfmt("URL literals are disabled"),
.msg = hintfmt("URL literals are disabled"),
.errPos = CUR_POS
});
$$ = new ExprString(data->symbols.create($1));
@ -478,7 +478,7 @@ binds
$$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data));
}
}
| { $$ = new ExprAttrs; }
| { $$ = new ExprAttrs(makeCurPos(@0, data)); }
;
attrs
@ -491,7 +491,7 @@ attrs
delete str;
} else
throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in inherit"),
.msg = hintfmt("dynamic attributes not allowed in inherit"),
.errPos = makeCurPos(@2, data)
});
}
@ -576,7 +576,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
ParseData data(*this);
data.origin = origin;
switch (origin) {
case foFile:
case foFile:
data.file = data.symbols.create(path);
break;
case foStdin:
@ -593,7 +593,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
int res = yyparse(scanner, &data);
yylex_destroy(scanner);
if (res) throw ParseError(data.error);
if (res) throw ParseError(data.error.value());
data.result->bindVars(staticEnv);
@ -614,8 +614,7 @@ Path resolveExprPath(Path path)
// Basic cycle/depth limit to avoid infinite loops.
if (++followCount >= maxFollow)
throw Error("too many symbolic links encountered while traversing the path '%s'", path);
if (lstat(path.c_str(), &st))
throw SysError("getting status of '%s'", path);
st = lstat(path);
if (!S_ISLNK(st.st_mode)) break;
path = absPath(readLink(path), dirOf(path));
}
@ -699,8 +698,12 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
Path res = r.second + suffix;
if (pathExists(res)) return canonPath(res);
}
if (hasPrefix(path, "nix/"))
return corepkgsPrefix + path.substr(4);
throw ThrownError({
.hint = hintfmt(evalSettings.pureEval
.msg = hintfmt(evalSettings.pureEval
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
path),
@ -722,8 +725,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
store, resolveUri(elem.second), "source", false).first.storePath) };
} catch (FileTransferError & e) {
logWarning({
.name = "Entry download",
.hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
});
res = { false, "" };
}
@ -733,8 +735,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
res = { true, path };
else {
logWarning({
.name = "Entry not found",
.hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
.msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
});
res = { false, "" };
}

1953
src/libexpr/primops.cc
View file

File diff suppressed because it is too large Load diff

10
src/libexpr/primops.hh
View file

@ -1,3 +1,5 @@
#pragma once
#include "eval.hh"
#include <tuple>
@ -10,9 +12,11 @@ struct RegisterPrimOp
struct Info
{
std::string name;
size_t arity;
PrimOpFun primOp;
std::vector<std::string> args;
size_t arity = 0;
const char * doc;
std::optional<std::string> requiredFeature;
PrimOpFun fun;
};
typedef std::vector<Info> PrimOps;
@ -26,6 +30,8 @@ struct RegisterPrimOp
size_t arity,
PrimOpFun fun,
std::optional<std::string> requiredFeature = {});
RegisterPrimOp(Info && info);
};
/* These primops are disabled without enableNativeCode, but plugins

16
src/libexpr/primops/context.cc
View file

@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos,
mkString(v, s, PathSet());
}
static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
@ -21,7 +21,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
mkBool(v, !context.empty());
}
static RegisterPrimOp r2("__hasContext", 1, prim_hasContext);
static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
@ -42,7 +42,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po
mkString(v, s, context2);
}
static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
/* Extract the context of a string as a structured Nix value.
@ -127,7 +127,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
v.attrs->sort();
}
static RegisterPrimOp r4("__getContext", 1, prim_getContext);
static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
/* Append the given context to a given string.
@ -147,7 +147,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
for (auto & i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name))
throw EvalError({
.hint = hintfmt("Context key '%s' is not a store path", i.name),
.msg = hintfmt("Context key '%s' is not a store path", i.name),
.errPos = *i.pos
});
if (!settings.readOnlyMode)
@ -164,7 +164,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) {
throw EvalError({
.hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
.msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
.errPos = *i.pos
});
}
@ -177,7 +177,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError({
.hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
.msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
.errPos = *i.pos
});
}
@ -191,6 +191,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
mkString(v, orig, context);
}
static RegisterPrimOp r5("__appendContext", 2, prim_appendContext);
static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext);
}

27
src/libexpr/primops/derivation.nix Normal file
View file

@ -0,0 +1,27 @@
/* This is the implementation of the ‘derivation’ builtin function.
It's actually a wrapper around the ‘derivationStrict’ primop. */
drvAttrs @ { outputs ? [ "out" ], ... }:
let
strict = derivationStrict drvAttrs;
commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) //
{ all = map (x: x.value) outputsList;
inherit drvAttrs;
};
outputToAttrListElement = outputName:
{ name = outputName;
value = commonAttrs // {
outPath = builtins.getAttr outputName strict;
drvPath = strict.drvPath;
type = "derivation";
inherit outputName;
};
};
outputsList = map outputToAttrListElement outputs;
in (builtins.head outputsList).value

11
src/libexpr/primops/fetchMercurial.cc
View file

@ -3,8 +3,7 @@
#include "store-api.hh"
#include "fetchers.hh"
#include "url.hh"
#include <regex>
#include "url-parts.hh"
namespace nix {
@ -18,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
@ -39,14 +38,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
.msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
.errPos = *attr.pos
});
}
if (url.empty())
throw EvalError({
.hint = hintfmt("'url' argument required"),
.msg = hintfmt("'url' argument required"),
.errPos = pos
});
@ -88,6 +87,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
state.allowedPaths->insert(tree.actualPath);
}
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);
static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);
}

214
src/libexpr/primops/fetchTree.cc
View file

@ -39,11 +39,12 @@ void emitTreeAttrs(
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
}
if (input.getType() == "git")
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
@ -84,26 +85,26 @@ static void fetchTree(
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
fetchers::Attrs attrs;
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
if (attr.value->type == tPath || attr.value->type == tString)
if (attr.value->type() == nPath || attr.value->type() == nString)
addURI(
state,
attrs,
attr.name,
state.coerceToString(*attr.pos, *attr.value, context, false, false)
);
else if (attr.value->type == tString)
else if (attr.value->type() == nString)
addURI(state, attrs, attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
else if (attr.value->type == tInt)
attrs.emplace(attr.name, attr.value->integer);
else if (attr.value->type() == nBool)
attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
else if (attr.value->type() == nInt)
attrs.emplace(attr.name, uint64_t(attr.value->integer));
else
throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
attr.name, showType(*attr.value));
@ -114,7 +115,7 @@ static void fetchTree(
if (!attrs.count("type"))
throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
@ -152,7 +153,8 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
fetchTree(state, pos, args, v, std::nullopt);
}
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
// FIXME: document
static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
const string & who, bool unpack, std::string name)
@ -162,7 +164,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
@ -176,14 +178,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
.msg = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
.errPos = *attr.pos
});
}
if (!url)
throw EvalError({
.hint = hintfmt("'url' argument required"),
.msg = hintfmt("'url' argument required"),
.errPos = pos
});
} else
@ -211,8 +213,8 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
@ -226,18 +228,192 @@ static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Va
fetch(state, pos, args, v, "fetchurl", false, "");
}
static RegisterPrimOp primop_fetchurl({
.name = "__fetchurl",
.args = {"url"},
.doc = R"(
Download the specified URL and return the path of the downloaded
file. This function is not available if [restricted evaluation
mode](../command-ref/conf-file.md) is enabled.
)",
.fun = prim_fetchurl,
});
static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetch(state, pos, args, v, "fetchTarball", true, "source");
}
static RegisterPrimOp primop_fetchTarball({
.name = "fetchTarball",
.args = {"args"},
.doc = R"(
Download the specified URL, unpack it and return the path of the
unpacked tree. The file must be a tape archive (`.tar`) compressed
with `gzip`, `bzip2` or `xz`. The top-level path component of the
files in the tarball is removed, so it is best if the tarball
contains a single directory at top level. The typical use of the
function is to obtain external Nix expression dependencies, such as
a particular version of Nixpkgs, e.g.
```nix
with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
stdenv.mkDerivation { … }
```
The fetched tarball is cached for a certain amount of time (1 hour
by default) in `~/.cache/nix/tarballs/`. You can change the cache
timeout either on the command line with `--option tarball-ttl number
of seconds` or in the Nix configuration file with this option: `
number of seconds to cache `.
Note that when obtaining the hash with ` nix-prefetch-url ` the
option `--unpack` is required.
This function can also verify the contents against a hash. In that
case, the function takes a set instead of a URL. The set requires
the attribute `url` and the attribute `sha256`, e.g.
```nix
with import (fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
}) {};
stdenv.mkDerivation { … }
```
This function is not available if [restricted evaluation
mode](../command-ref/conf-file.md) is enabled.
)",
.fun = prim_fetchTarball,
});
static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
{
fetchTree(state, pos, args, v, "git", true);
}
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
static RegisterPrimOp primop_fetchGit({
.name = "fetchGit",
.args = {"args"},
.doc = R"(
Fetch a path from git. *args* can be a URL, in which case the HEAD
of the repo at that URL is fetched. Otherwise, it can be an
attribute with the following attributes (all except `url` optional):
- url\
The URL of the repo.
- name\
The name of the directory the repo should be exported to in the
store. Defaults to the basename of the URL.
- rev\
The git revision to fetch. Defaults to the tip of `ref`.
- ref\
The git ref to look for the requested revision under. This is
often a branch or tag name. Defaults to `HEAD`.
By default, the `ref` value is prefixed with `refs/heads/`. As
of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts
with `refs/`.
- submodules\
A Boolean parameter that specifies whether submodules should be
checked out. Defaults to `false`.
- allRefs\
Whether to fetch all refs of the repository. With this argument being
true, it's possible to load a `rev` from *any* `ref` (by default only
`rev`s from the specified `ref` are supported).
Here are some examples of how to use `fetchGit`.
- To fetch a private repository over SSH:
```nix
builtins.fetchGit {
url = "git@github.com:my-secret/repository.git";
ref = "master";
rev = "adab8b916a45068c044658c4158d81878f9ed1c3";
}
```
- To fetch an arbitrary reference:
```nix
builtins.fetchGit {
url = "https://github.com/NixOS/nix.git";
ref = "refs/heads/0.5-release";
}
```
- If the revision you're looking for is in the default branch of
the git repository you don't strictly need to specify the branch
name in the `ref` attribute.
However, if the revision you're looking for is in a future
branch for the non-default branch you will need to specify the
the `ref` attribute as well.
```nix
builtins.fetchGit {
url = "https://github.com/nixos/nix.git";
rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
ref = "1.11-maintenance";
}
```
> **Note**
>
> It is nice to always specify the branch which a revision
> belongs to. Without the branch being specified, the fetcher
> might fail if the default branch changes. Additionally, it can
> be confusing to try a commit from a non-default branch and see
> the fetch fail. If the branch is specified the fault is much
> more obvious.
- If the revision you're looking for is in the default branch of
the git repository you may omit the `ref` attribute.
```nix
builtins.fetchGit {
url = "https://github.com/nixos/nix.git";
rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452";
}
```
- To fetch a specific tag:
```nix
builtins.fetchGit {
url = "https://github.com/nixos/nix.git";
ref = "refs/tags/1.9";
}
```
- To fetch the latest version of a remote branch:
```nix
builtins.fetchGit {
url = "ssh://git@github.com/nixos/nix.git";
ref = "master";
}
```
> **Note**
>
> Nix will refetch the branch in accordance with
> the option `tarball-ttl`.
> **Note**
>
> This behavior is disabled in *Pure evaluation mode*.
)",
.fun = prim_fetchGit,
});
}

4
src/libexpr/primops/fromTOML.cc
View file

@ -82,12 +82,12 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
visit(v, parser(tomlStream).parse());
} catch (std::runtime_error & e) {
throw EvalError({
.hint = hintfmt("while parsing a TOML string: %s", e.what()),
.msg = hintfmt("while parsing a TOML string: %s", e.what()),
.errPos = pos
});
}
}
static RegisterPrimOp r("fromTOML", 1, prim_fromTOML);
static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML);
}

25
src/libexpr/value-to-json.cc
View file

@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict,
if (strict) state.forceValue(v);
switch (v.type) {
switch (v.type()) {
case tInt:
case nInt:
out.write(v.integer);
break;
case tBool:
case nBool:
out.write(v.boolean);
break;
case tString:
case nString:
copyContext(v, context);
out.write(v.string.s);
break;
case tPath:
case nPath:
out.write(state.copyPathToStore(context, v.path));
break;
case tNull:
case nNull:
out.write(nullptr);
break;
case tAttrs: {
case nAttrs: {
auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
if (maybeString) {
out.write(*maybeString);
@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
}
case tList1: case tList2: case tListN: {
case nList: {
auto list(out.list());
for (unsigned int n = 0; n < v.listSize(); ++n) {
auto placeholder(list.placeholder());
@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
}
case tExternal:
case nExternal:
v.external->printValueAsJSON(state, strict, out, context);
break;
case tFloat:
case nFloat:
out.write(v.fpoint);
break;
default:
case nThunk:
throw TypeError("cannot convert %1% to JSON", showType(v));
case nFunction:
throw TypeError("cannot convert %1% to JSON", showType(v));
}
}

33
src/libexpr/value-to-xml.cc
View file

@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
if (strict) state.forceValue(v);
switch (v.type) {
switch (v.type()) {
case tInt:
case nInt:
doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
break;
case tBool:
case nBool:
doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
break;
case tString:
case nString:
/* !!! show the context? */
copyContext(v, context);
doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
break;
case tPath:
case nPath:
doc.writeEmptyElement("path", singletonAttrs("value", v.path));
break;
case tNull:
case nNull:
doc.writeEmptyElement("null");
break;
case tAttrs:
case nAttrs:
if (state.isDerivation(v)) {
XMLAttrs xmlAttrs;
@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
a = v.attrs->find(state.sDrvPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
if (a->value->type == tString)
if (a->value->type() == nString)
xmlAttrs["drvPath"] = drvPath = a->value->string.s;
}
a = v.attrs->find(state.sOutPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
if (a->value->type == tString)
if (a->value->type() == nString)
xmlAttrs["outPath"] = a->value->string.s;
}
@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
break;
case tList1: case tList2: case tListN: {
case nList: {
XMLOpenElement _(doc, "list");
for (unsigned int n = 0; n < v.listSize(); ++n)
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
break;
}
case tLambda: {
case nFunction: {
if (!v.isLambda()) {
// FIXME: Serialize primops and primopapps
doc.writeEmptyElement("unevaluated");
break;
}
XMLAttrs xmlAttrs;
if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
XMLOpenElement _(doc, "function", xmlAttrs);
@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
break;
}
case tExternal:
case nExternal:
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
break;
case tFloat:
case nFloat:
doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
break;
default:
case nThunk:
doc.writeEmptyElement("unevaluated");
}
}

246
src/libexpr/value.hh
View file

@ -27,8 +27,24 @@ typedef enum {
tPrimOpApp,
tExternal,
tFloat
} ValueType;
} InternalType;
// This type abstracts over all actual value types in the language,
// grouping together implementation details like tList*, different function
// types, and types in non-normal form (so thunks and co.)
typedef enum {
nThunk,
nInt,
nFloat,
nBool,
nString,
nPath,
nNull,
nAttrs,
nList,
nFunction,
nExternal
} ValueType;
class Bindings;
struct Env;
@ -90,7 +106,28 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
struct Value
{
ValueType type;
private:
InternalType internalType;
friend std::string showType(const Value & v);
friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
public:
// Functions needed to distinguish the type
// These should be removed eventually, by putting the functionality that's
// needed by callers into methods of this type
// type() == nThunk
inline bool isThunk() const { return internalType == tThunk; };
inline bool isApp() const { return internalType == tApp; };
inline bool isBlackhole() const { return internalType == tBlackhole; };
// type() == nFunction
inline bool isLambda() const { return internalType == tLambda; };
inline bool isPrimOp() const { return internalType == tPrimOp; };
inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
union
{
NixInt integer;
@ -147,26 +184,165 @@ struct Value
NixFloat fpoint;
};
// Returns the normal type of a Value. This only returns nThunk if the
// Value hasn't been forceValue'd
inline ValueType type() const
{
switch (internalType) {
case tInt: return nInt;
case tBool: return nBool;
case tString: return nString;
case tPath: return nPath;
case tNull: return nNull;
case tAttrs: return nAttrs;
case tList1: case tList2: case tListN: return nList;
case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
case tExternal: return nExternal;
case tFloat: return nFloat;
case tThunk: case tApp: case tBlackhole: return nThunk;
}
abort();
}
/* After overwriting an app node, be sure to clear pointers in the
Value to ensure that the target isn't kept alive unnecessarily. */
inline void clearValue()
{
app.left = app.right = 0;
}
inline void mkInt(NixInt n)
{
clearValue();
internalType = tInt;
integer = n;
}
inline void mkBool(bool b)
{
clearValue();
internalType = tBool;
boolean = b;
}
inline void mkString(const char * s, const char * * context = 0)
{
internalType = tString;
string.s = s;
string.context = context;
}
inline void mkPath(const char * s)
{
clearValue();
internalType = tPath;
path = s;
}
inline void mkNull()
{
clearValue();
internalType = tNull;
}
inline void mkAttrs(Bindings * a)
{
clearValue();
internalType = tAttrs;
attrs = a;
}
inline void mkList(size_t size)
{
clearValue();
if (size == 1)
internalType = tList1;
else if (size == 2)
internalType = tList2;
else {
internalType = tListN;
bigList.size = size;
}
}
inline void mkThunk(Env * e, Expr * ex)
{
internalType = tThunk;
thunk.env = e;
thunk.expr = ex;
}
inline void mkApp(Value * l, Value * r)
{
internalType = tApp;
app.left = l;
app.right = r;
}
inline void mkLambda(Env * e, ExprLambda * f)
{
internalType = tLambda;
lambda.env = e;
lambda.fun = f;
}
inline void mkBlackhole()
{
internalType = tBlackhole;
// Value will be overridden anyways
}
inline void mkPrimOp(PrimOp * p)
{
clearValue();
internalType = tPrimOp;
primOp = p;
}
inline void mkPrimOpApp(Value * l, Value * r)
{
internalType = tPrimOpApp;
app.left = l;
app.right = r;
}
inline void mkExternal(ExternalValueBase * e)
{
clearValue();
internalType = tExternal;
external = e;
}
inline void mkFloat(NixFloat n)
{
clearValue();
internalType = tFloat;
fpoint = n;
}
bool isList() const
{
return type == tList1 || type == tList2 || type == tListN;
return internalType == tList1 || internalType == tList2 || internalType == tListN;
}
Value * * listElems()
{
return type == tList1 || type == tList2 ? smallList : bigList.elems;
return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
}
const Value * const * listElems() const
{
return type == tList1 || type == tList2 ? smallList : bigList.elems;
return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
}
size_t listSize() const
{
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size;
}
Pos determinePos(const Pos &pos) const;
/* Check whether forcing this value requires a trivial amount of
computation. In particular, function applications are
non-trivial. */
@ -176,86 +352,42 @@ struct Value
};
/* After overwriting an app node, be sure to clear pointers in the
Value to ensure that the target isn't kept alive unnecessarily. */
static inline void clearValue(Value & v)
{
v.app.left = v.app.right = 0;
}
// TODO: Remove these static functions, replace call sites with v.mk* instead
static inline void mkInt(Value & v, NixInt n)
{
clearValue(v);
v.type = tInt;
v.integer = n;
v.mkInt(n);
}
static inline void mkFloat(Value & v, NixFloat n)
{
clearValue(v);
v.type = tFloat;
v.fpoint = n;
v.mkFloat(n);
}
static inline void mkBool(Value & v, bool b)
{
clearValue(v);
v.type = tBool;
v.boolean = b;
v.mkBool(b);
}
static inline void mkNull(Value & v)
{
clearValue(v);
v.type = tNull;
v.mkNull();
}
static inline void mkApp(Value & v, Value & left, Value & right)
{
v.type = tApp;
v.app.left = &left;
v.app.right = &right;
v.mkApp(&left, &right);
}
static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
{
v.type = tPrimOpApp;
v.app.left = &left;
v.app.right = &right;
}
static inline void mkStringNoCopy(Value & v, const char * s)
{
v.type = tString;
v.string.s = s;
v.string.context = 0;
}
static inline void mkString(Value & v, const Symbol & s)
{
mkStringNoCopy(v, ((const string &) s).c_str());
v.mkString(((const string &) s).c_str());
}
void mkString(Value & v, const char * s);
static inline void mkPathNoCopy(Value & v, const char * s)
{
clearValue(v);
v.type = tPath;
v.path = s;
}
void mkPath(Value & v, const char * s);

8
src/libfetchers/attrs.cc
View file

@ -11,11 +11,11 @@ Attrs jsonToAttrs(const nlohmann::json & json)
for (auto & i : json.items()) {
if (i.value().is_number())
attrs.emplace(i.key(), i.value().get<int64_t>());
attrs.emplace(i.key(), i.value().get<uint64_t>());
else if (i.value().is_string())
attrs.emplace(i.key(), i.value().get<std::string>());
else if (i.value().is_boolean())
attrs.emplace(i.key(), i.value().get<bool>());
attrs.emplace(i.key(), Explicit<bool> { i.value().get<bool>() });
else
throw Error("unsupported input attribute type in lock file");
}
@ -23,7 +23,7 @@ Attrs jsonToAttrs(const nlohmann::json & json)
return attrs;
}
nlohmann::json attrsToJson(const Attrs & attrs)
nlohmann::json attrsToJSON(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
@ -44,7 +44,7 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump());
}
std::string getStrAttr(const Attrs & attrs, const std::string & name)

14
src/libfetchers/attrs.hh
View file

@ -8,24 +8,12 @@
namespace nix::fetchers {
/* Wrap bools to prevent string literals (i.e. 'char *') from being
cast to a bool in Attr. */
template<typename T>
struct Explicit {
T t;
bool operator ==(const Explicit<T> & other) const
{
return t == other.t;
}
};
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
nlohmann::json attrsToJson(const Attrs & attrs);
nlohmann::json attrsToJSON(const Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);

20
src/libfetchers/cache.cc
View file

@ -55,8 +55,8 @@ struct CacheImpl : Cache
bool immutable) override
{
_state.lock()->add.use()
(attrsToJson(inAttrs).dump())
(attrsToJson(infoAttrs).dump())
(attrsToJSON(inAttrs).dump())
(attrsToJSON(infoAttrs).dump())
(store->printStorePath(storePath))
(immutable)
(time(0)).exec();
@ -70,7 +70,7 @@ struct CacheImpl : Cache
if (!res->expired)
return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
debug("ignoring expired cache entry '%s'",
attrsToJson(inAttrs).dump());
attrsToJSON(inAttrs).dump());
}
return {};
}
@ -81,15 +81,15 @@ struct CacheImpl : Cache
{
auto state(_state.lock());
auto inAttrsJson = attrsToJson(inAttrs).dump();
auto inAttrsJSON = attrsToJSON(inAttrs).dump();
auto stmt(state->lookup.use()(inAttrsJson));
auto stmt(state->lookup.use()(inAttrsJSON));
if (!stmt.next()) {
debug("did not find cache entry for '%s'", inAttrsJson);
debug("did not find cache entry for '%s'", inAttrsJSON);
return {};
}
auto infoJson = stmt.getStr(0);
auto infoJSON = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1));
auto immutable = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
@ -97,16 +97,16 @@ struct CacheImpl : Cache
store->addTempRoot(storePath);
if (!store->isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s'", inAttrsJson);
debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
return {};
}
debug("using cache entry '%s' -> '%s', '%s'",
inAttrsJson, infoJson, store->printStorePath(storePath));
inAttrsJSON, infoJSON, store->printStorePath(storePath));
return Result {
.expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
.storePath = std::move(storePath)
};
}

27
src/libfetchers/fetchers.cc
View file

@ -65,10 +65,18 @@ Input Input::fromAttrs(Attrs && attrs)
ParsedURL Input::toURL() const
{
if (!scheme)
throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
throw Error("cannot show unsupported input '%s'", attrsToJSON(attrs));
return scheme->toURL(*this);
}
std::string Input::toURLString(const std::map<std::string, std::string> & extraQuery) const
{
auto url = toURL();
for (auto & attr : extraQuery)
url.query.insert(attr);
return url.to_string();
}
std::string Input::to_string() const
{
return toURL().to_string();
@ -102,7 +110,7 @@ bool Input::contains(const Input & other) const
std::pair<Tree, Input> Input::fetch(ref<Store> store) const
{
if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
/* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the
@ -124,18 +132,25 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
}
}
auto [tree, input] = scheme->fetch(store, *this);
auto [tree, input] = [&]() -> std::pair<Tree, Input> {
try {
return scheme->fetch(store, *this);
} catch (Error & e) {
e.addTrace({}, "while fetching the input '%s'", to_string());
throw;
}
}();
if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath);
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash)
throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
}
if (auto prevLastModified = getLastModified()) {
@ -239,7 +254,7 @@ std::optional<time_t> Input::getLastModified() const
ParsedURL InputScheme::toURL(const Input & input)
{
throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs));
}
Input InputScheme::applyOverrides(

33
src/libfetchers/fetchers.hh
View file

@ -21,9 +21,17 @@ struct Tree
struct InputScheme;
/* The Input object is generated by a specific fetcher, based on the
* user-supplied input attribute in the flake.nix file, and contains
* the information that the specific fetcher needs to perform the
* actual fetch. The Input object is most commonly created via the
* "fromURL()" or "fromAttrs()" static functions which are provided
* the url or attrset specified in the flake file.
*/
struct Input
{
friend class InputScheme;
friend struct InputScheme;
std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;
@ -39,6 +47,8 @@ public:
ParsedURL toURL() const;
std::string toURLString(const std::map<std::string, std::string> & extraQuery = {}) const;
std::string to_string() const;
Attrs toAttrs() const;
@ -73,7 +83,7 @@ public:
StorePath computeStorePath(Store & store) const;
// Convience functions for common attributes.
// Convenience functions for common attributes.
std::string getType() const;
std::optional<Hash> getNarHash() const;
std::optional<std::string> getRef() const;
@ -82,8 +92,21 @@ public:
std::optional<time_t> getLastModified() const;
};
/* The InputScheme represents a type of fetcher. Each fetcher
* registers with nix at startup time. When processing an input for a
* flake, each scheme is given an opportunity to "recognize" that
* input from the url or attributes in the flake file's specification
* and return an Input object to represent the input if it is
* recognized. The Input object contains the information the fetcher
* needs to actually perform the "fetch()" when called.
*/
struct InputScheme
{
virtual ~InputScheme()
{ }
virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
@ -119,12 +142,14 @@ DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable);
bool immutable,
const Headers & headers = {});
std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable);
bool immutable,
const Headers & headers = {});
}

71
src/libfetchers/git.cc
View file

@ -3,8 +3,10 @@
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <sys/time.h>
#include <sys/wait.h>
using namespace std::string_literals;
@ -58,12 +60,13 @@ struct GitInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules");
maybeGetBoolAttr(attrs, "allRefs");
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (std::regex_search(*ref, badGitRefRegex))
@ -151,12 +154,14 @@ struct GitInputScheme : InputScheme
std::pair<bool, std::string> getActualUrl(const Input & input) const
{
// Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or
// HEAD).
// file:// URIs are normally not cloned (but otherwise treated the
// same as remote URIs, i.e. we don't use the working tree or
// HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
// repo, treat as a remote URI to force a clone.
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file" && !forceHttp;
bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
return {isLocal, isLocal ? url.path : url.base};
}
@ -168,10 +173,12 @@ struct GitInputScheme : InputScheme
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
if (submodules) cacheType += "-submodules";
if (allRefs) cacheType += "-all-refs";
auto getImmutableAttrs = [&]()
{
@ -269,10 +276,10 @@ struct GitInputScheme : InputScheme
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {
Tree(store->printStorePath(storePath), std::move(storePath)),
Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
}
@ -337,11 +344,15 @@ struct GitInputScheme : InputScheme
}
}
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
if (allRefs) {
doFetch = true;
} else {
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
}
}
if (doFetch) {
@ -351,9 +362,13 @@ struct GitInputScheme : InputScheme
// we're using --quiet for now. Should process its stderr.
try {
auto ref = input.getRef();
auto fetchRef = ref->compare(0, 5, "refs/") == 0
? *ref
: "refs/heads/" + *ref;
auto fetchRef = allRefs
? "refs/*"
: ref->compare(0, 5, "refs/") == 0
? *ref
: ref == "HEAD"
? *ref
: "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
@ -391,6 +406,28 @@ struct GitInputScheme : InputScheme
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
RunOptions checkCommitOpts(
"git",
{ "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
);
checkCommitOpts.searchPath = true;
checkCommitOpts.mergeStderrToStdout = true;
auto result = runProgram(checkCommitOpts);
if (WEXITSTATUS(result.first) == 128
&& result.second.find("bad file") != std::string::npos
) {
throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
"Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
"allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
input.getRev()->gitRev(),
*input.getRef(),
actualUrl
);
}
if (submodules) {
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);
@ -421,7 +458,7 @@ struct GitInputScheme : InputScheme
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
@ -451,6 +488,6 @@ struct GitInputScheme : InputScheme
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}

171
src/libfetchers/github.cc
View file

@ -3,19 +3,30 @@
#include "fetchers.hh"
#include "globals.hh"
#include "store-api.hh"
#include "types.hh"
#include "url-parts.hh"
#include <optional>
#include <nlohmann/json.hpp>
namespace nix::fetchers {
// A github or gitlab url
const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
struct DownloadUrl
{
std::string url;
Headers headers;
};
// A github or gitlab host
const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
struct GitArchiveInputScheme : InputScheme
{
virtual std::string type() = 0;
virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != type()) return {};
@ -26,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme
std::optional<std::string> ref;
std::optional<std::string> host_url;
if (path.size() == 2) {
} else if (path.size() == 3) {
auto size = path.size();
if (size == 3) {
if (std::regex_match(path[2], revRegex))
rev = Hash::parseAny(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
} else
} else if (size > 3) {
std::string rs;
for (auto i = std::next(path.begin(), 2); i != path.end(); i++) {
rs += *i;
if (std::next(i) != path.end()) {
rs += "/";
}
}
if (std::regex_match(rs, refRegex)) {
ref = rs;
} else {
throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs);
}
} else if (size < 2)
throw BadURL("URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
@ -50,9 +75,9 @@ struct GitArchiveInputScheme : InputScheme
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
ref = value;
}
else if (name == "url") {
if (!std::regex_match(value, urlRegex))
throw BadURL("URL '%s' contains an invalid instance url", url.url);
else if (name == "host") {
if (!std::regex_match(value, hostRegex))
throw BadURL("URL '%s' contains an invalid instance host", url.url);
host_url = value;
}
// FIXME: barf on unsupported attributes
@ -67,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
if (host_url) input.attrs.insert_or_assign("url", *host_url);
if (host_url) input.attrs.insert_or_assign("host", *host_url);
return input;
}
@ -77,7 +102,7 @@ struct GitArchiveInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != type()) return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
throw Error("unsupported input attribute '%s'", name);
getStrAttr(attrs, "owner");
@ -129,9 +154,31 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
std::optional<std::string> getAccessToken(const std::string & host) const
{
auto tokens = settings.accessTokens.get();
if (auto token = get(tokens, host))
return *token;
return {};
}
Headers makeHeadersWithAuthTokens(const std::string & host) const
{
Headers headers;
auto accessToken = getAccessToken(host);
if (accessToken) {
auto hdr = accessHeaderFromToken(*accessToken);
if (hdr)
headers.push_back(*hdr);
else
warn("Unrecognized access token for host '%s'", host);
}
return headers;
}
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
virtual std::string getDownloadUrl(const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
@ -160,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
auto [tree, lastModified] = downloadTarball(store, url, "source", true);
auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
input.attrs.insert_or_assign("lastModified", lastModified);
input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
{"lastModified", lastModified}
{"lastModified", uint64_t(lastModified)}
},
tree.storePath,
true);
@ -182,41 +229,52 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
std::string type() override { return "github"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Github supports PAT/OAuth2 tokens and HTTP Basic
// Authentication. The former simply specifies the token, the
// latter can use the token as the password. Only the first
// is used here. See
// https://developer.github.com/v3/#authentication and
// https://docs.github.com/en/developers/apps/authorizing-oath-apps
return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
std::string getDownloadUrl(const Input & input) const override
DownloadUrl getDownloadUrl(const Input & input) const override
{
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
url += "?access_token=" + accessToken;
return url;
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers };
}
void clone(const Input & input, const Path & destDir) override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
}
@ -226,48 +284,71 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
std::string type() override { return "gitlab"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Gitlab supports 4 kinds of authorization, two of which are
// relevant here: OAuth2 and PAT (Private Access Token). The
// user can indicate which token is used by specifying the
// token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
// If the <TYPE> is unrecognized, this will fall back to
// treating this simply has <HDRNAME>:<HDRVAL>. See
// https://docs.gitlab.com/12.10/ee/api/README.html#authentication
auto fldsplit = token.find_first_of(':');
// n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
if ("OAuth2" == token.substr(0, fldsplit))
return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
if ("PAT" == token.substr(0, fldsplit))
return std::make_pair("Private-token", token.substr(fldsplit+1));
warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit));
return std::nullopt;
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
std::string getDownloadUrl(const Input & input) const override
DownloadUrl getDownloadUrl(const Input & input) const override
{
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
// This endpoint has a rate limit threshold that may be
// server-specific and vary based whether the user is
// authenticated via an accessToken or not, but the usual rate
// is 10 reqs/sec/ip-addr. See
// https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
/* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
url += "?access_token=" + accessToken;*/
return url;
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers };
}
void clone(const Input & input, const Path & destDir) override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
}

3
src/libfetchers/indirect.cc
View file

@ -1,4 +1,5 @@
#include "fetchers.hh"
#include "url-parts.hh"
namespace nix::fetchers {
@ -99,6 +100,6 @@ struct IndirectInputScheme : InputScheme
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
}

61
src/libfetchers/mercurial.cc
View file

@ -3,6 +3,7 @@
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include <sys/time.h>
@ -10,6 +11,36 @@ using namespace std::string_literals;
namespace nix::fetchers {
namespace {
RunOptions hgOptions(const Strings & args) {
RunOptions opts("hg", args);
opts.searchPath = true;
auto env = getEnv();
// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
env["HGPLAIN"] = "";
opts.environment = env;
return opts;
}
// runProgram wrapper that uses hgOptions instead of stock RunOptions.
string runHg(const Strings & args, const std::optional<std::string> & input = {})
{
RunOptions opts = hgOptions(args);
opts.input = input;
auto res = runProgram(opts);
if (!statusOk(res.first))
throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
return res.second;
}
}
struct MercurialInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) override
@ -99,11 +130,11 @@ struct MercurialInputScheme : InputScheme
assert(sourcePath);
// FIXME: shut up if file is already tracked.
runProgram("hg", true,
runHg(
{ "add", *sourcePath + "/" + std::string(file) });
if (commitMsg)
runProgram("hg", true,
runHg(
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
}
@ -129,7 +160,7 @@ struct MercurialInputScheme : InputScheme
if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
if (!clean) {
@ -142,10 +173,10 @@ struct MercurialInputScheme : InputScheme
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
@ -165,7 +196,7 @@ struct MercurialInputScheme : InputScheme
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {
Tree(store->printStorePath(storePath), std::move(storePath)),
Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
}
@ -223,33 +254,33 @@ struct MercurialInputScheme : InputScheme
if (!(input.getRev()
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
runHg({ "pull", "-R", cacheDir, "--", actualUrl });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
runHg({ "recover", "-R", cacheDir });
runHg({ "pull", "-R", cacheDir, "--", actualUrl });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
runHg({ "clone", "--noupdate", "--", actualUrl, cacheDir });
}
}
auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
@ -262,7 +293,7 @@ struct MercurialInputScheme : InputScheme
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
@ -270,7 +301,7 @@ struct MercurialInputScheme : InputScheme
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
{"revCount", (int64_t) revCount},
{"revCount", (uint64_t) revCount},
});
if (!_input.getRev())
@ -292,6 +323,6 @@ struct MercurialInputScheme : InputScheme
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
}

8
src/libfetchers/path.cc
View file

@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme
if (name == "rev" || name == "narHash")
input.attrs.insert_or_assign(name, value);
else if (name == "revCount" || name == "lastModified") {
uint64_t n;
if (!string2Int(value, n))
if (auto n = string2Int<uint64_t>(value))
input.attrs.insert_or_assign(name, *n);
else
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input.attrs.insert_or_assign(name, n);
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
@ -102,6 +102,6 @@ struct PathInputScheme : InputScheme
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
}

11
src/libfetchers/registry.cc
View file

@ -3,6 +3,7 @@
#include "util.hh"
#include "globals.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include <nlohmann/json.hpp>
@ -59,10 +60,10 @@ void Registry::write(const Path & path)
nlohmann::json arr;
for (auto & entry : entries) {
nlohmann::json obj;
obj["from"] = attrsToJson(entry.from.toAttrs());
obj["to"] = attrsToJson(entry.to.toAttrs());
obj["from"] = attrsToJSON(entry.from.toAttrs());
obj["to"] = attrsToJSON(entry.to.toAttrs());
if (!entry.extraAttrs.empty())
obj["to"].update(attrsToJson(entry.extraAttrs));
obj["to"].update(attrsToJSON(entry.extraAttrs));
if (entry.exact)
obj["exact"] = true;
arr.emplace_back(std::move(obj));
@ -113,7 +114,7 @@ static std::shared_ptr<Registry> getSystemRegistry()
Path getUserRegistryPath()
{
return getHome() + "/.config/nix/registry.json";
return getConfigDir() + "/nix/registry.json";
}
std::shared_ptr<Registry> getUserRegistry()
@ -147,7 +148,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
if (!hasPrefix(path, "/")) {
auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
path = store->toRealPath(storePath);
}

21
src/libfetchers/tarball.cc
View file

@ -5,6 +5,7 @@
#include "store-api.hh"
#include "archive.hh"
#include "tarfile.hh"
#include "types.hh"
namespace nix::fetchers {
@ -12,7 +13,8 @@ DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable)
bool immutable,
const Headers & headers)
{
// FIXME: check store
@ -37,6 +39,7 @@ DownloadFileResult downloadFile(
return useCached();
FileTransferRequest request(url);
request.headers = headers;
if (cached)
request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
FileTransferResult res;
@ -61,14 +64,15 @@ DownloadFileResult downloadFile(
if (res.cached) {
assert(cached);
assert(request.expectedETag == res.etag);
storePath = std::move(cached->storePath);
} else {
StringSink sink;
dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
hashString(htSHA256, *sink.s),
};
info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
@ -109,7 +113,8 @@ std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable)
bool immutable,
const Headers & headers)
{
Attrs inAttrs({
{"type", "tarball"},
@ -125,7 +130,7 @@ std::pair<Tree, time_t> downloadTarball(
getIntAttr(cached->infoAttrs, "lastModified")
};
auto res = downloadFile(store, url, name, immutable);
auto res = downloadFile(store, url, name, immutable, headers);
std::optional<StorePath> unpackedStorePath;
time_t lastModified;
@ -146,7 +151,7 @@ std::pair<Tree, time_t> downloadTarball(
}
Attrs infoAttrs({
{"lastModified", lastModified},
{"lastModified", uint64_t(lastModified)},
{"etag", res.etag},
});
@ -225,6 +230,6 @@ struct TarballInputScheme : InputScheme
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
}

28
src/libmain/common-args.cc
View file

@ -10,25 +10,28 @@ MixCommonArgs::MixCommonArgs(const string & programName)
addFlag({
.longName = "verbose",
.shortName = 'v',
.description = "increase verbosity level",
.description = "Increase the logging verbosity level.",
.category = loggingCategory,
.handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
});
addFlag({
.longName = "quiet",
.description = "decrease verbosity level",
.description = "Decrease the logging verbosity level.",
.category = loggingCategory,
.handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
});
addFlag({
.longName = "debug",
.description = "enable debug output",
.description = "Set the logging verbosity level to 'debug'.",
.category = loggingCategory,
.handler = {[]() { verbosity = lvlDebug; }},
});
addFlag({
.longName = "option",
.description = "set a Nix configuration option (overriding nix.conf)",
.description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
.labels = {"name", "value"},
.handler = {[](std::string name, std::string value) {
try {
@ -44,15 +47,15 @@ MixCommonArgs::MixCommonArgs(const string & programName)
globalConfig.getSettings(settings);
for (auto & s : settings)
if (hasPrefix(s.first, prefix))
completions->insert(s.first);
completions->add(s.first, fmt("Set the `%s` setting.", s.first));
}
}
});
addFlag({
.longName = "log-format",
.description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
"or \"bar-with-logs\"",
.description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.",
.category = loggingCategory,
.labels = {"format"},
.handler = {[](std::string format) { setLogFormat(format); }},
});
@ -60,14 +63,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
addFlag({
.longName = "max-jobs",
.shortName = 'j',
.description = "maximum number of parallel builds",
.description = "The maximum number of parallel builds.",
.labels = Strings{"jobs"},
.handler = {[=](std::string s) {
settings.set("max-jobs", s);
}}
});
std::string cat = "config";
std::string cat = "Options to override configuration settings";
globalConfig.convertToArgs(*this, cat);
// Backward compatibility hack: nix-env already had a --system flag.
@ -76,4 +79,11 @@ MixCommonArgs::MixCommonArgs(const string & programName)
hiddenCategories.insert(cat);
}
void MixCommonArgs::initialFlagsProcessed()
{
initPlugins();
pluginsInited();
}
}

23
src/libmain/common-args.hh
View file

@ -4,10 +4,17 @@
namespace nix {
struct MixCommonArgs : virtual Args
//static constexpr auto commonArgsCategory = "Miscellaneous common options";
static constexpr auto loggingCategory = "Logging-related options";
class MixCommonArgs : public virtual Args
{
void initialFlagsProcessed() override;
public:
string programName;
MixCommonArgs(const string & programName);
protected:
virtual void pluginsInited() {}
};
struct MixDryRun : virtual Args
@ -16,7 +23,12 @@ struct MixDryRun : virtual Args
MixDryRun()
{
mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun);
addFlag({
.longName = "dry-run",
.description = "Show what this command would do without doing it.",
//.category = commonArgsCategory,
.handler = {&dryRun, true},
});
}
};
@ -26,7 +38,12 @@ struct MixJSON : virtual Args
MixJSON()
{
mkFlag(0, "json", "produce JSON output", &json);
addFlag({
.longName = "json",
.description = "Produce output in JSON format, suitable for consumption by another program.",
//.category = commonArgsCategory,
.handler = {&json, true},
});
}
};

4
src/libmain/loggers.cc
View file

@ -12,7 +12,7 @@ LogFormat parseLogFormat(const std::string & logFormatStr) {
else if (logFormatStr == "raw-with-logs")
return LogFormat::rawWithLogs;
else if (logFormatStr == "internal-json")
return LogFormat::internalJson;
return LogFormat::internalJSON;
else if (logFormatStr == "bar")
return LogFormat::bar;
else if (logFormatStr == "bar-with-logs")
@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
return makeSimpleLogger(false);
case LogFormat::rawWithLogs:
return makeSimpleLogger(true);
case LogFormat::internalJson:
case LogFormat::internalJSON:
return makeJSONLogger(*makeSimpleLogger(true));
case LogFormat::bar:
return makeProgressBar();

2
src/libmain/loggers.hh
View file

@ -7,7 +7,7 @@ namespace nix {
enum class LogFormat {
raw,
rawWithLogs,
internalJson,
internalJSON,
bar,
barWithLogs,
};

16
src/libmain/progress-bar.cc
View file

@ -122,6 +122,7 @@ public:
void log(Verbosity lvl, const FormatOrString & fs) override
{
if (lvl > verbosity) return;
auto state(state_.lock());
log(*state, lvl, fs.s);
}
@ -256,7 +257,7 @@ public:
}
else if (type == resBuildLogLine || type == resPostBuildLogLine) {
auto lastLine = trim(getS(fields, 0));
auto lastLine = chomp(getS(fields, 0));
if (!lastLine.empty()) {
auto i = state->its.find(act);
assert(i != state->its.end());
@ -362,7 +363,7 @@ public:
auto width = getWindowSize().second;
if (width <= 0) width = std::numeric_limits<decltype(width)>::max();
writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
}
std::string getStatus(State & state)
@ -466,6 +467,17 @@ public:
Logger::writeToStdout(s);
}
}
std::optional<char> ask(std::string_view msg) override
{
auto state(state_.lock());
if (!state->active || !isatty(STDIN_FILENO)) return {};
std::cerr << fmt("\r\e[K%s ", msg);
auto s = trim(readLine(STDIN_FILENO));
if (s.size() != 1) return {};
draw(*state);
return s[0];
}
};
Logger * makeProgressBar(bool printBuildLogs)

71
src/libmain/shared.cc
View file

@ -18,6 +18,8 @@
#include <openssl/crypto.h>
#include <sodium.h>
namespace nix {
@ -34,7 +36,7 @@ void printGCWarning()
}
void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
void printMissing(ref<Store> store, const std::vector<DerivedPath> & paths, Verbosity lvl)
{
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
@ -126,6 +128,9 @@ void initNix()
CRYPTO_set_locking_callback(opensslLockCallback);
#endif
if (sodium_init() == -1)
throw Error("could not initialise libsodium");
loadConfFile();
startSignalHandlerThread();
@ -181,50 +186,64 @@ LegacyArgs::LegacyArgs(const std::string & programName,
addFlag({
.longName = "no-build-output",
.shortName = 'Q',
.description = "do not show build output",
.description = "Do not show build output.",
.handler = {[&]() {setLogFormat(LogFormat::raw); }},
});
addFlag({
.longName = "keep-failed",
.shortName ='K',
.description = "keep temporary directories of failed builds",
.description = "Keep temporary directories of failed builds.",
.handler = {&(bool&) settings.keepFailed, true},
});
addFlag({
.longName = "keep-going",
.shortName ='k',
.description = "keep going after a build fails",
.description = "Keep going after a build fails.",
.handler = {&(bool&) settings.keepGoing, true},
});
addFlag({
.longName = "fallback",
.description = "build from source if substitution fails",
.description = "Build from source if substitution fails.",
.handler = {&(bool&) settings.tryFallback, true},
});
auto intSettingAlias = [&](char shortName, const std::string & longName,
const std::string & description, const std::string & dest) {
mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
settings.set(dest, std::to_string(n));
const std::string & description, const std::string & dest)
{
addFlag({
.longName = longName,
.shortName = shortName,
.description = description,
.labels = {"n"},
.handler = {[=](std::string s) {
auto n = string2IntWithUnitPrefix<uint64_t>(s);
settings.set(dest, std::to_string(n));
}}
});
};
intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores");
intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time");
intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout");
mkFlag(0, "readonly-mode", "do not write to the Nix store",
&settings.readOnlyMode);
addFlag({
.longName = "readonly-mode",
.description = "Do not write to the Nix store.",
.handler = {&settings.readOnlyMode, true},
});
mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
&gcWarning, false);
addFlag({
.longName = "no-gc-warning",
.description = "Disable warnings about not using `--add-root`.",
.handler = {&gcWarning, true},
});
addFlag({
.longName = "store",
.description = "URI of the Nix store to use",
.description = "The URL of the Nix store to use.",
.labels = {"store-uri"},
.handler = {&(std::string&) settings.storeUri},
});
@ -274,9 +293,9 @@ void printVersion(const string & programName)
#if HAVE_BOEHMGC
cfg.push_back("gc");
#endif
#if HAVE_SODIUM
cfg.push_back("signed-caches");
#endif
std::cout << "System type: " << settings.thisSystem << "\n";
std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
std::cout << "User configuration files: " <<
@ -291,7 +310,7 @@ void printVersion(const string & programName)
void showManPage(const string & name)
{
restoreSignals();
restoreProcessContext();
setenv("MANPATH", settings.nixManDir.c_str(), 1);
execlp("man", "man", name.c_str(), nullptr);
throw SysError("command 'man %1%' failed", name.c_str());
@ -354,7 +373,7 @@ RunPager::RunPager()
throw SysError("dupping stdin");
if (!getenv("LESS"))
setenv("LESS", "FRSXMK", 1);
restoreSignals();
restoreProcessContext();
if (pager)
execl("/bin/sh", "sh", "-c", pager, nullptr);
execlp("pager", "pager", nullptr);
@ -384,18 +403,12 @@ RunPager::~RunPager()
}
string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
PrintFreed::~PrintFreed()
{
if (show)
std::cout << format("%1% store paths deleted, %2% freed\n")
% results.paths.size()
% showBytes(results.bytesFreed);
std::cout << fmt("%d store paths deleted, %s freed\n",
results.paths.size(),
showBytes(results.bytesFreed));
}
Exit::~Exit() { }

21
src/libmain/shared.hh
View file

@ -4,6 +4,7 @@
#include "args.hh"
#include "common-args.hh"
#include "path.hh"
#include "derived-path.hh"
#include <signal.h>
@ -42,7 +43,7 @@ struct StorePathWithOutputs;
void printMissing(
ref<Store> store,
const std::vector<StorePathWithOutputs> & paths,
const std::vector<DerivedPath> & paths,
Verbosity lvl = lvlInfo);
void printMissing(ref<Store> store, const StorePathSet & willBuild,
@ -57,23 +58,7 @@ template<class N> N getIntArg(const string & opt,
{
++i;
if (i == end) throw UsageError("'%1%' requires an argument", opt);
string s = *i;
N multiplier = 1;
if (allowUnit && !s.empty()) {
char u = std::toupper(*s.rbegin());
if (std::isalpha(u)) {
if (u == 'K') multiplier = 1ULL << 10;
else if (u == 'M') multiplier = 1ULL << 20;
else if (u == 'G') multiplier = 1ULL << 30;
else if (u == 'T') multiplier = 1ULL << 40;
else throw UsageError("invalid unit specifier '%1%'", u);
s.resize(s.size() - 1);
}
}
N n;
if (!string2Int(s, n))
throw UsageError("'%1%' requires an integer argument", opt);
return n * multiplier;
return string2IntWithUnitPrefix<N>(*i);
}

161
src/libstore/binary-cache-store.cc
View file

@ -11,6 +11,7 @@
#include "nar-accessor.hh"
#include "json.hh"
#include "thread-pool.hh"
#include "callback.hh"
#include <chrono>
#include <future>
@ -22,7 +23,8 @@
namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params)
: Store(params)
: BinaryCacheStoreConfig(params)
, Store(params)
{
if (secretKeyFile != "")
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
@ -84,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
promise.set_exception(std::current_exception());
}
}});
auto data = promise.get_future().get();
sink((unsigned char *) data->data(), data->size());
sink(*promise.get_future().get());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
@ -140,17 +141,10 @@ struct FileSource : FdSource
}
};
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
std::function<ValidPathInfo(HashResult)> mkInfo)
{
assert(info.narHash && info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
auto [fdTemp, fnTemp] = createTempFile();
AutoDelete autoDelete(fnTemp);
@ -160,13 +154,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink(htSHA256);
HashSink fileHashSink { htSHA256 };
std::shared_ptr<FSAccessor> narAccessor;
HashSink narHashSink { htSHA256 };
{
FdSink fileSink(fdTemp.get());
TeeSink teeSink(fileSink, fileHashSink);
auto compressionSink = makeCompressionSink(compression, teeSink);
TeeSource teeSource(narSource, *compressionSink);
TeeSink teeSinkCompressed { fileSink, fileHashSink };
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
TeeSource teeSource { narSource, teeSinkUncompressed };
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
fileSink.flush();
@ -174,9 +170,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
auto now2 = std::chrono::steady_clock::now();
auto info = mkInfo(narHashSink.finish());
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = info.narSize;
narInfo->narHash = info.narHash;
narInfo->compression = compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
@ -184,6 +179,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "zstd" ? ".zst" :
compression == "lzip" ? ".lzip" :
compression == "lz4" ? ".lz4" :
compression == "br" ? ".br" :
"");
@ -219,7 +217,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}
}
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
}
/* Optionally maintain an index of DWARF debug info files
@ -298,6 +296,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
writeNarInfo(narInfo);
stats.narInfoWrite++;
return narInfo;
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
/* FIXME reinstate these, once we can correctly do hash modulo sink as
needed. We need to throw here in case we uploaded a corrupted store path. */
// assert(info.narHash == nar.first);
// assert(info.narSize == nar.second);
return info;
}});
}
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name),
nar.first,
};
info.narSize = nar.second;
return info;
})->path;
}
bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
@ -312,14 +345,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
{
auto info = queryPathInfo(storePath).cast<const NarInfo>();
uint64_t narSize = 0;
LengthSink narSize;
TeeSink tee { sink, narSize };
LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
sink(data, len);
narSize += len;
});
auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
auto decompressor = makeDecompressionSink(info->compression, tee);
try {
getFile(info->url, *decompressor);
@ -331,7 +360,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
stats.narRead++;
//stats.narReadCompressedBytes += nar->size(); // FIXME
stats.narReadBytes += narSize;
stats.narReadBytes += narSize.length;
}
void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
@ -369,44 +398,72 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
// FIXME: some cut&paste from LocalStore::addToStore().
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default
implementation of this method in terms of addToStoreFromDump. */
/* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
std::optional<Hash> h;
HashSink sink { hashAlgo };
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
} else {
auto s = readFile(srcPath);
dumpString(s, sink);
h = hashString(hashAlgo, s);
readFile(srcPath, sink);
}
auto h = sink.finish().first;
ValidPathInfo info(makeFixedOutputPath(method, *h, name));
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs);
return std::move(info.path);
auto source = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink, filter);
});
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
makeFixedOutputPath(method, h, name),
nar.first,
};
info.narSize = nar.second;
info.ca = FixedOutputHash {
.method = method,
.hash = h,
};
return info;
})->path;
}
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
ValidPathInfo info(computeStorePathForText(name, s, references));
info.references = references;
auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, textHash, references);
if (repair || !isValidPath(info.path)) {
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs);
if (!repair && isValidPath(path))
return path;
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first };
info.narSize = nar.second;
info.ca = TextHash { textHash };
info.references = references;
return info;
})->path;
}
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
{
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) {
return {Realisation::fromJSON(
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
} else {
return std::nullopt;
}
}
return std::move(info.path);
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
upsertFile(filePath, info.toJSON().dump(), "application/json");
}
ref<FSAccessor> BinaryCacheStore::getFSAccessor()

43
src/libstore/binary-cache-store.hh
View file

@ -11,17 +11,21 @@ namespace nix {
struct NarInfo;
class BinaryCacheStore : public Store
struct BinaryCacheStoreConfig : virtual StoreConfig
{
public:
using StoreConfig::StoreConfig;
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression",
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
};
class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
{
private:
@ -29,6 +33,9 @@ private:
protected:
// The prefix under which realisation infos will be stored
const std::string realisationsPrefix = "/realisations";
BinaryCacheStore(const Params & params);
public:
@ -58,7 +65,7 @@ public:
public:
virtual void init();
virtual void init() override;
private:
@ -68,6 +75,10 @@ private:
void writeNarInfo(ref<NarInfo> narInfo);
ref<const ValidPathInfo> addToStoreCommon(
Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
std::function<ValidPathInfo(HashResult)> mkInfo);
public:
bool isValidPathUncached(const StorePath & path) override;
@ -81,6 +92,9 @@ public:
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
@ -88,15 +102,12 @@ public:
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void narFromPath(const StorePath & path, Sink & sink) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override
{ unsupported("buildDerivation"); }
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
ref<FSAccessor> getFSAccessor() override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;

5206
src/libstore/build.cc
View file

File diff suppressed because it is too large Load diff

1328
src/libstore/build/derivation-goal.cc Normal file
View file

File diff suppressed because it is too large Load diff

232
src/libstore/build/derivation-goal.hh Normal file
View file

@ -0,0 +1,232 @@
#pragma once
#include "parsed-derivations.hh"
#include "lock.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "goal.hh"
namespace nix {
using std::map;
struct HookInstance;
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
/* Unless we are repairing, we don't both to test validity and just assume it,
so the choices are `Absent` or `Valid`. */
enum struct PathStatus {
Corrupt,
Absent,
Valid,
};
struct InitialOutputStatus {
StorePath path;
PathStatus status;
/* Valid in the store, and additionally non-corrupt if we are repairing */
bool isValid() const {
return status == PathStatus::Valid;
}
/* Merely present, allowed to be corrupt */
bool isPresent() const {
return status == PathStatus::Corrupt
|| status == PathStatus::Valid;
}
};
struct InitialOutput {
bool wanted;
Hash outputHash;
std::optional<InitialOutputStatus> known;
};
struct DerivationGoal : public Goal
{
/* Whether to use an on-disk .drv file. */
bool useDerivation;
/* The path of the derivation. */
StorePath drvPath;
/* The path of the corresponding resolved derivation */
std::optional<BasicDerivation> resolvedDrv;
/* The specific outputs that we need to build. Empty means all of
them. */
StringSet wantedOutputs;
/* Whether additional wanted outputs have been added. */
bool needRestart = false;
/* Whether to retry substituting the outputs after building the
inputs. */
bool retrySubstitution;
/* The derivation stored at drvPath. */
std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv;
/* The remainder is state held during the build. */
/* Locks on (fixed) output paths. */
PathLocks outputLocks;
/* All input paths (that is, the union of FS closures of the
immediate input paths). */
StorePathSet inputPaths;
std::map<std::string, InitialOutput> initialOutputs;
/* File descriptor for the log file. */
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
/* Number of bytes received from the builder's stdout/stderr. */
unsigned long logSize;
/* The most recent log lines. */
std::list<std::string> logTail;
std::string currentLogLine;
size_t currentLogLinePos = 0; // to handle carriage return
std::string currentHookLine;
/* The build hook. */
std::unique_ptr<HookInstance> hook;
/* The sort of derivation we are building. */
DerivationType derivationType;
typedef void (DerivationGoal::*GoalState)();
GoalState state;
/* The final output paths of the build.
- For input-addressed derivations, always the precomputed paths
- For content-addressed derivations, calcuated from whatever the hash
ends up being. (Note that fixed outputs derivations that produce the
"wrong" output still install that data under its true content-address.)
*/
OutputPathMap finalOutputs;
BuildMode buildMode;
BuildResult result;
/* The current round, if we're building multiple times. */
size_t curRound = 1;
size_t nrRounds;
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act;
/* Activity that denotes waiting for a lock. */
std::unique_ptr<Activity> actLock;
std::map<ActivityId, Activity> builderActivities;
/* The remote machine on which we're building. */
std::string machineName;
DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
virtual ~DerivationGoal();
void timedOut(Error && ex) override;
string key() override;
void work() override;
/* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs);
BuildResult getResult() { return result; }
/* The states. */
void getDerivation();
void loadDerivation();
void haveDerivation();
void outputsSubstitutionTried();
void gaveUpOnSubstitution();
void closureRepaired();
void inputsRealised();
void tryToBuild();
virtual void tryLocalBuild();
void buildDone();
void resolvedFinished();
/* Is the build hook willing to perform the build? */
HookReply tryBuildHook();
virtual int getChildStatus();
/* Check that the derivation outputs all exist and register them
as valid. */
virtual void registerOutputs();
/* Open a log file and a pipe to it. */
Path openLogFile();
/* Sign the newly built realisation if the store allows it */
virtual void signRealisation(Realisation&) {}
/* Close the log file. */
void closeLogFile();
/* Close the read side of the logger pipe. */
virtual void closeReadPipes();
/* Cleanup hooks for buildDone() */
virtual void cleanupHookFinally();
virtual void cleanupPreChildKill();
virtual void cleanupPostChildKill();
virtual bool cleanupDecideWhetherDiskFull();
virtual void cleanupPostOutputsRegisteredModeCheck();
virtual void cleanupPostOutputsRegisteredModeNonCheck();
virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
void flushLine();
/* Wrappers around the corresponding Store methods that first consult the
derivation. This is currently needed because when there is no drv file
there also is no DB entry. */
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap();
/* Return the set of (in)valid paths. */
void checkPathValidity();
/* Forcibly kill the child process, if any. */
virtual void killChild();
void repairClosure();
void started();
void done(
BuildResult::Status status,
std::optional<Error> ex = {});
StorePathSet exportReferences(const StorePathSet & storePaths);
};
MakeError(NotDeterministic, BuildError);
}

95
src/libstore/build/drv-output-substitution-goal.cc Normal file
View file

@ -0,0 +1,95 @@
#include "drv-output-substitution-goal.hh"
#include "worker.hh"
#include "substitution-goal.hh"
namespace nix {
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, id(id)
{
state = &DrvOutputSubstitutionGoal::init;
name = fmt("substitution of '%s'", id.to_string());
trace("created");
}
void DrvOutputSubstitutionGoal::init()
{
trace("init");
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext();
}
void DrvOutputSubstitutionGoal::tryNext()
{
trace("Trying next substituter");
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
if (substituterFailed) {
worker.failedSubstitutions++;
worker.updateProgress();
}
return;
}
auto sub = subs.front();
subs.pop_front();
// FIXME: Make async
outputInfo = sub->queryRealisation(id);
if (!outputInfo) {
tryNext();
return;
}
addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
if (waitees.empty()) outPathValid();
else state = &DrvOutputSubstitutionGoal::outPathValid;
}
void DrvOutputSubstitutionGoal::outPathValid()
{
assert(outputInfo);
trace("Output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
return;
}
worker.store.registerDrvOutput(*outputInfo);
finished();
}
void DrvOutputSubstitutionGoal::finished()
{
trace("finished");
amDone(ecSuccess);
}
string DrvOutputSubstitutionGoal::key()
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(id.to_string());
}
void DrvOutputSubstitutionGoal::work()
{
(this->*state)();
}
}

50
src/libstore/build/drv-output-substitution-goal.hh Normal file
View file

@ -0,0 +1,50 @@
#pragma once
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
namespace nix {
class Worker;
// Substitution of a derivation output.
// This is done in three steps:
// 1. Fetch the output info from a substituter
// 2. Substitute the corresponding output path
// 3. Register the output info
class DrvOutputSubstitutionGoal : public Goal {
private:
// The drv output we're trying to substitue
DrvOutput id;
// The realisation corresponding to the given output id.
// Will be filled once we can get it.
std::optional<Realisation> outputInfo;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* Whether a substituter failed. */
bool substituterFailed = false;
public:
DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
typedef void (DrvOutputSubstitutionGoal::*GoalState)();
GoalState state;
void init();
void tryNext();
void outPathValid();
void finished();
void timedOut(Error && ex) override { abort(); };
string key() override;
void work() override;
};
}

133
src/libstore/build/entry-points.cc Normal file
View file

@ -0,0 +1,133 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
#include "local-store.hh"
namespace nix {
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
{
Worker worker(*this);
Goals goals;
for (auto & br : reqs) {
std::visit(overloaded {
[&](DerivedPath::Built bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](DerivedPath::Opaque bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
}
worker.run(goals);
StorePathSet failed;
std::optional<Error> ex;
for (auto & i : goals) {
if (i->ex) {
if (ex)
logError(i->ex->info());
else
ex = i->ex;
}
if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
}
}
if (failed.size() == 1 && ex) {
ex->status = worker.exitStatus();
throw *ex;
} else if (!failed.empty()) {
if (ex) logError(ex->info());
throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
}
}
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
Worker worker(*this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
BuildResult result;
try {
worker.run(Goals{goal});
result = goal->getResult();
} catch (Error & e) {
result.status = BuildResult::MiscFailure;
result.errorMsg = e.msg();
}
// XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
// extended to return the full realisation for each output
auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
auto outputHashes = staticOutputHashes(*this, drv);
for (auto & [outputName, staticOutput] : staticDrvOutputs) {
auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
if (staticOutput.second)
result.builtOutputs.insert_or_assign(
outputId,
Realisation{ outputId, *staticOutput.second}
);
if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
auto realisation = this->queryRealisation(outputId);
if (realisation)
result.builtOutputs.insert_or_assign(
outputId,
*realisation
);
}
}
return result;
}
void Store::ensurePath(const StorePath & path)
{
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
Worker worker(*this);
GoalPtr goal = worker.makePathSubstitutionGoal(path);
Goals goals = {goal};
worker.run(goals);
if (goal->exitCode != Goal::ecSuccess) {
if (goal->ex) {
goal->ex->status = worker.exitStatus();
throw *goal->ex;
} else
throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
}
}
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
Goals goals = {goal};
worker.run(goals);
if (goal->exitCode != Goal::ecSuccess) {
/* Since substituting the path didn't work, if we have a valid
deriver, then rebuild the deriver. */
auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) {
goals.clear();
goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair));
worker.run(goals);
} else
throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path));
}
}
}

91
src/libstore/build/goal.cc Normal file
View file

@ -0,0 +1,91 @@
#include "goal.hh"
#include "worker.hh"
namespace nix {
bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
string s1 = a->key();
string s2 = b->key();
return s1 < s2;
}
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
// FIXME: necessary?
// FIXME: O(n)
for (auto & i : goals)
if (i.lock() == p) return;
goals.push_back(p);
}
void Goal::addWaitee(GoalPtr waitee)
{
waitees.insert(waitee);
addToWeakGoals(waitee->waiters, shared_from_this());
}
void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
{
assert(waitees.find(waitee) != waitees.end());
waitees.erase(waitee);
trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
if (result == ecNoSubstituters) ++nrNoSubstituters;
if (result == ecIncompleteClosure) ++nrIncompleteClosure;
if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & goal : waitees) {
WeakGoals waiters2;
for (auto & j : goal->waiters)
if (j.lock() != shared_from_this()) waiters2.push_back(j);
goal->waiters = waiters2;
}
waitees.clear();
worker.wakeUp(shared_from_this());
}
}
void Goal::amDone(ExitCode result, std::optional<Error> ex)
{
trace("done");
assert(exitCode == ecBusy);
assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
exitCode = result;
if (ex) {
if (!waiters.empty())
logError(ex->info());
else
this->ex = std::move(*ex);
}
for (auto & i : waiters) {
GoalPtr goal = i.lock();
if (goal) goal->waiteeDone(shared_from_this(), result);
}
waiters.clear();
worker.removeGoal(shared_from_this());
cleanup();
}
void Goal::trace(const FormatOrString & fs)
{
debug("%1%: %2%", name, fs.s);
}
}

109
src/libstore/build/goal.hh Normal file
View file

@ -0,0 +1,109 @@
#pragma once
#include "types.hh"
#include "store-api.hh"
namespace nix {
/* Forward definition. */
struct Goal;
class Worker;
/* A pointer to a goal. */
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
struct CompareGoalPtrs {
bool operator() (const GoalPtr & a, const GoalPtr & b) const;
};
/* Set of goals. */
typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef list<WeakGoalPtr> WeakGoals;
/* A map of paths to goals (and the other way around). */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
struct Goal : public std::enable_shared_from_this<Goal>
{
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
/* Backlink to the worker. */
Worker & worker;
/* Goals that this goal is waiting for. */
Goals waitees;
/* Goals waiting for this one to finish. Must use weak pointers
here to prevent cycles. */
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
unsigned int nrFailed;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
unsigned int nrNoSubstituters;
/* Number of substitution goals we are/were waiting for that
failed because they had unsubstitutable references. */
unsigned int nrIncompleteClosure;
/* Name of this goal for debugging purposes. */
string name;
/* Whether the goal is finished. */
ExitCode exitCode;
/* Exception containing an error message, if any. */
std::optional<Error> ex;
Goal(Worker & worker) : worker(worker)
{
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
exitCode = ecBusy;
}
virtual ~Goal()
{
trace("goal destroyed");
}
virtual void work() = 0;
void addWaitee(GoalPtr waitee);
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
virtual void handleChildOutput(int fd, const string & data)
{
abort();
}
virtual void handleEOF(int fd)
{
abort();
}
void trace(const FormatOrString & fs);
string getName()
{
return name;
}
/* Callback in case of a timeout. It should wake up its waiters,
get rid of any running child processes that are being monitored
by the worker (important!), etc. */
virtual void timedOut(Error && ex) = 0;
virtual string key() = 0;
void amDone(ExitCode result, std::optional<Error> ex = {});
virtual void cleanup() { }
};
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
}

72
src/libstore/build/hook-instance.cc Normal file
View file

@ -0,0 +1,72 @@
#include "globals.hh"
#include "hook-instance.hh"
namespace nix {
HookInstance::HookInstance()
{
debug("starting build hook '%s'", settings.buildHook);
/* Create a pipe to get the output of the child. */
fromHook.create();
/* Create the communication pipes. */
toHook.create();
/* Create a pipe to get the output of the builder. */
builderOut.create();
/* Fork the hook. */
pid = startProcess([&]() {
commonChildInit(fromHook);
if (chdir("/") == -1) throw SysError("changing into /");
/* Dup the communication pipes. */
if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping to-hook read side");
/* Use fd 4 for the builder's stdout/stderr. */
if (dup2(builderOut.writeSide.get(), 4) == -1)
throw SysError("dupping builder's stdout/stderr");
/* Hack: pass the read side of that fd to allow build-remote
to read SSH error messages. */
if (dup2(builderOut.readSide.get(), 5) == -1)
throw SysError("dupping builder's stdout/stderr");
Strings args = {
std::string(baseNameOf(settings.buildHook.get())),
std::to_string(verbosity),
};
execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
throw SysError("executing '%s'", settings.buildHook);
});
pid.setSeparatePG(true);
fromHook.writeSide = -1;
toHook.readSide = -1;
sink = FdSink(toHook.writeSide.get());
std::map<std::string, Config::SettingInfo> settings;
globalConfig.getSettings(settings);
for (auto & setting : settings)
sink << 1 << setting.first << setting.second.value;
sink << 0;
}
HookInstance::~HookInstance()
{
try {
toHook.writeSide = -1;
if (pid != -1) pid.kill();
} catch (...) {
ignoreException();
}
}
}

31
src/libstore/build/hook-instance.hh Normal file
View file

@ -0,0 +1,31 @@
#pragma once
#include "logging.hh"
#include "serialise.hh"
namespace nix {
struct HookInstance
{
/* Pipes for talking to the build hook. */
Pipe toHook;
/* Pipe for the hook's standard output/error. */
Pipe fromHook;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* The process ID of the hook. */
Pid pid;
FdSink sink;
std::map<ActivityId, Activity> activities;
HookInstance();
~HookInstance();
};
}

2861
src/libstore/build/local-derivation-goal.cc Normal file
View file

File diff suppressed because it is too large Load diff

202
src/libstore/build/local-derivation-goal.hh Normal file
View file

@ -0,0 +1,202 @@
#pragma once
#include "derivation-goal.hh"
#include "local-store.hh"
namespace nix {
struct LocalDerivationGoal : public DerivationGoal
{
LocalStore & getLocalStore();
/* User selected for running the builder. */
std::unique_ptr<UserLock> buildUser;
/* The process ID of the builder. */
Pid pid;
/* The temporary directory. */
Path tmpDir;
/* The path of the temporary directory in the sandbox. */
Path tmpDirInSandbox;
/* Pipe for the builder's standard output/error. */
Pipe builderOut;
/* Pipe for synchronising updates to the builder namespaces. */
Pipe userNamespaceSync;
/* The mount namespace of the builder, used to add additional
paths to the sandbox as a result of recursive Nix calls. */
AutoCloseFD sandboxMountNamespace;
/* On Linux, whether we're doing the build in its own user
namespace. */
bool usingUserNamespace = true;
/* Whether we're currently doing a chroot build. */
bool useChroot = false;
Path chrootRootDir;
/* RAII object to delete the chroot directory. */
std::shared_ptr<AutoDelete> autoDelChroot;
/* Whether to run the build in a private network namespace. */
bool privateNetwork = false;
/* Stuff we need to pass to initChild(). */
struct ChrootPath {
Path source;
bool optional;
ChrootPath(Path source = "", bool optional = false)
: source(source), optional(optional)
{ }
};
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
DirsInChroot dirsInChroot;
typedef map<string, string> Environment;
Environment env;
#if __APPLE__
typedef string SandboxProfile;
SandboxProfile additionalSandboxProfile;
#endif
/* Hash rewriting. */
StringMap inputRewrites, outputRewrites;
typedef map<StorePath, StorePath> RedirectedOutputs;
RedirectedOutputs redirectedOutputs;
/* The outputs paths used during the build.
- Input-addressed derivations or fixed content-addressed outputs are
sometimes built when some of their outputs already exist, and can not
be hidden via sandboxing. We use temporary locations instead and
rewrite after the build. Otherwise the regular predetermined paths are
put here.
- Floating content-addressed derivations do not know their final build
output paths until the outputs are hashed, so random locations are
used, and then renamed. The randomness helps guard against hidden
self-references.
*/
OutputPathMap scratchOutputs;
/* Path registration info from the previous round, if we're
building multiple times. Since this contains the hash, it
allows us to compare whether two rounds produced the same
result. */
std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); }
const static Path homeDir;
/* The recursive Nix daemon socket. */
AutoCloseFD daemonSocket;
/* The daemon main thread. */
std::thread daemonThread;
/* The daemon worker threads. */
std::vector<std::thread> daemonWorkerThreads;
/* Paths that were added via recursive Nix calls. */
StorePathSet addedPaths;
/* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
/nix/store/<bla> is some arbitrary path in a binary cache). */
bool isAllowed(const StorePath & path)
{
return inputPaths.count(path) || addedPaths.count(path);
}
bool isAllowed(const DerivedPath & req);
friend struct RestrictedStore;
using DerivationGoal::DerivationGoal;
virtual ~LocalDerivationGoal() override;
/* Whether we need to perform hash rewriting if there are valid output paths. */
bool needsHashRewrite();
/* The additional states. */
void tryLocalBuild() override;
/* Start building a derivation. */
void startBuilder();
/* Fill in the environment for the builder. */
void initEnv();
/* Setup tmp dir location. */
void initTmpDir();
/* Write a JSON file containing the derivation attributes. */
void writeStructuredAttrs();
void startDaemon();
void stopDaemon();
/* Add 'path' to the set of paths that may be referenced by the
outputs, and make it appear in the sandbox. */
void addDependency(const StorePath & path);
/* Make a file owned by the builder. */
void chownToBuilder(const Path & path);
int getChildStatus() override;
/* Run the builder's process. */
void runChild();
/* Check that the derivation outputs all exist and register them
as valid. */
void registerOutputs() override;
void signRealisation(Realisation &) override;
/* Check that an output meets the requirements specified by the
'outputChecks' attribute (or the legacy
'{allowed,disallowed}{References,Requisites}' attributes). */
void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
/* Close the read side of the logger pipe. */
void closeReadPipes() override;
/* Cleanup hooks for buildDone() */
void cleanupHookFinally() override;
void cleanupPreChildKill() override;
void cleanupPostChildKill() override;
bool cleanupDecideWhetherDiskFull() override;
void cleanupPostOutputsRegisteredModeCheck() override;
void cleanupPostOutputsRegisteredModeNonCheck() override;
bool isReadDesc(int fd) override;
/* Delete the temporary directory, if we have one. */
void deleteTmpDir(bool force);
/* Forcibly kill the child process, if any. */
void killChild() override;
/* Create alternative path calculated from but distinct from the
input, so we can avoid overwriting outputs (or other store paths)
that already exist. */
StorePath makeFallbackPath(const StorePath & path);
/* Make a path to another based on the output name along with the
derivation hash. */
/* FIXME add option to randomize, so we can audit whether our
rewrites caught everything */
StorePath makeFallbackPath(std::string_view outputName);
};
}

302
src/libstore/build/substitution-goal.cc Normal file
View file

@ -0,0 +1,302 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "nar-info.hh"
#include "finally.hh"
namespace nix {
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
, ca(ca)
{
state = &PathSubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
trace("created");
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
}
PathSubstitutionGoal::~PathSubstitutionGoal()
{
cleanup();
}
void PathSubstitutionGoal::work()
{
(this->*state)();
}
void PathSubstitutionGoal::init()
{
trace("init");
worker.store.addTempRoot(storePath);
/* If the path already exists we're done. */
if (!repair && worker.store.isValidPath(storePath)) {
amDone(ecSuccess);
return;
}
if (settings.readOnlyMode)
throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext();
}
void PathSubstitutionGoal::tryNext()
{
trace("trying next substituter");
cleanup();
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
if (substituterFailed) {
worker.failedSubstitutions++;
worker.updateProgress();
}
return;
}
sub = subs.front();
subs.pop_front();
if (ca) {
subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) {
tryNext();
return;
}
try {
// FIXME: make async
info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) {
tryNext();
return;
} catch (SubstituterDisabled &) {
if (settings.tryFallback) {
tryNext();
return;
}
throw;
} catch (Error & e) {
if (settings.tryFallback) {
logError(e.info());
tryNext();
return;
}
throw;
}
if (info->path != storePath) {
if (info->isContentAddressed(*sub) && info->references.empty()) {
auto info2 = std::make_shared<ValidPathInfo>(*info);
info2->path = storePath;
info = info2;
} else {
printError("asked '%s' for '%s' but got '%s'",
sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
tryNext();
return;
}
}
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
maintainExpectedDownload =
narInfo && narInfo->fileSize
? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
: nullptr;
worker.updateProgress();
/* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */
if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
{
warn("substituter '%s' does not have a valid signature for path '%s'",
sub->getUri(), worker.store.printStorePath(storePath));
tryNext();
return;
}
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
addWaitee(worker.makePathSubstitutionGoal(i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
referencesValid();
else
state = &PathSubstitutionGoal::referencesValid;
}
void PathSubstitutionGoal::referencesValid()
{
trace("all references realised");
if (nrFailed > 0) {
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
return;
}
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
assert(worker.store.isValidPath(i));
state = &PathSubstitutionGoal::tryToRun;
worker.wakeUp(shared_from_this());
}
void PathSubstitutionGoal::tryToRun()
{
trace("trying to run");
/* Make sure that we are allowed to start a build. Note that even
if maxBuildJobs == 0 (no local builds allowed), we still allow
a substituter to run. This is because substitutions cannot be
distributed to another machine via the build hook. */
if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
worker.waitForBuildSlot(shared_from_this());
return;
}
maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
worker.updateProgress();
outPipe.create();
promise = std::promise<void>();
thr = std::thread([this]() {
try {
/* Wake up the worker loop when we're done. */
Finally updateStats([this]() { outPipe.writeSide.close(); });
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
promise.set_exception(std::current_exception());
}
});
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
state = &PathSubstitutionGoal::finished;
}
void PathSubstitutionGoal::finished()
{
trace("substitute finished");
thr.join();
worker.childTerminated(this);
try {
promise.get_future().get();
} catch (std::exception & e) {
printError(e.what());
/* Cause the parent build to fail unless --fallback is given,
or the substitute has disappeared. The latter case behaves
the same as the substitute never having existed in the
first place. */
try {
throw;
} catch (SubstituteGone &) {
} catch (...) {
substituterFailed = true;
}
/* Try the next substitute. */
state = &PathSubstitutionGoal::tryNext;
worker.wakeUp(shared_from_this());
return;
}
worker.markContentsGood(storePath);
printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath));
maintainRunningSubstitutions.reset();
maintainExpectedSubstitutions.reset();
worker.doneSubstitutions++;
if (maintainExpectedDownload) {
auto fileSize = maintainExpectedDownload->delta;
maintainExpectedDownload.reset();
worker.doneDownloadSize += fileSize;
}
worker.doneNarSize += maintainExpectedNar->delta;
maintainExpectedNar.reset();
worker.updateProgress();
amDone(ecSuccess);
}
void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
{
}
void PathSubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}
void PathSubstitutionGoal::cleanup()
{
try {
if (thr.joinable()) {
// FIXME: signal worker thread to quit.
thr.join();
worker.childTerminated(this);
}
outPipe.close();
} catch (...) {
ignoreException();
}
}
}

86
src/libstore/build/substitution-goal.hh Normal file
View file

@ -0,0 +1,86 @@
#pragma once
#include "lock.hh"
#include "store-api.hh"
#include "goal.hh"
namespace nix {
class Worker;
struct PathSubstitutionGoal : public Goal
{
/* The store path that should be realised through a substitute. */
StorePath storePath;
/* The path the substituter refers to the path as. This will be
different when the stores have different names. */
std::optional<StorePath> subPath;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* The current substituter. */
std::shared_ptr<Store> sub;
/* Whether a substituter failed. */
bool substituterFailed = false;
/* Path info returned by the substituter's query info operation. */
std::shared_ptr<const ValidPathInfo> info;
/* Pipe for the substituter's standard output. */
Pipe outPipe;
/* The substituter thread. */
std::thread thr;
std::promise<void> promise;
/* Whether to try to repair a valid path. */
RepairFlag repair;
/* Location where we're downloading the substitute. Differs from
storePath when doing a repair. */
Path destPath;
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
typedef void (PathSubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
public:
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~PathSubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
string key() override
{
/* "a$" ensures substitution goals happen before derivation
goals. */
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
}
void work() override;
/* The states. */
void init();
void tryNext();
void gotInfo();
void referencesValid();
void tryToRun();
void finished();
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
void cleanup() override;
};
}

497
src/libstore/build/worker.cc Normal file
View file

@ -0,0 +1,497 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
#include <poll.h>
namespace nix {
Worker::Worker(Store & store)
: act(*logger, actRealise)
, actDerivations(*logger, actBuilds)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
{
/* Debugging: prevent recursive workers. */
nrLocalBuilds = 0;
lastWokenUp = steady_time_point::min();
permanentFailure = false;
timedOut = false;
hashMismatch = false;
checkMismatch = false;
}
Worker::~Worker()
{
/* Explicitly get rid of all strong pointers now. After this all
goals that refer to this worker should be gone. (Otherwise we
are in trouble, since goals may call childTerminated() etc. in
their destructors). */
topGoals.clear();
assert(expectedSubstitutions == 0);
assert(expectedDownloadSize == 0);
assert(expectedNarSize == 0);
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
const StringSet & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
{
std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
if (!goal) {
goal = mkDrvGoal();
goal_weak = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
return goal;
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
});
}
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
});
}
std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<PathSubstitutionGoal>(path, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
{
std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
i != goalMap.end(); )
if (i->second.lock() == goal) {
auto j = i; ++j;
goalMap.erase(i);
i = j;
}
else ++i;
}
void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
else
assert(false);
if (topGoals.find(goal) != topGoals.end()) {
topGoals.erase(goal);
/* If a top-level goal failed, then kill all other goals
(unless keepGoing was set). */
if (goal->exitCode == Goal::ecFailed && !settings.keepGoing)
topGoals.clear();
}
/* Wake up goals waiting for any goal to finish. */
for (auto & i : waitingForAnyGoal) {
GoalPtr goal = i.lock();
if (goal) wakeUp(goal);
}
waitingForAnyGoal.clear();
}
void Worker::wakeUp(GoalPtr goal)
{
goal->trace("woken up");
addToWeakGoals(awake, goal);
}
unsigned Worker::getNrLocalBuilds()
{
return nrLocalBuilds;
}
void Worker::childStarted(GoalPtr goal, const set<int> & fds,
bool inBuildSlot, bool respectTimeouts)
{
Child child;
child.goal = goal;
child.goal2 = goal.get();
child.fds = fds;
child.timeStarted = child.lastOutput = steady_time_point::clock::now();
child.inBuildSlot = inBuildSlot;
child.respectTimeouts = respectTimeouts;
children.emplace_back(child);
if (inBuildSlot) nrLocalBuilds++;
}
void Worker::childTerminated(Goal * goal, bool wakeSleepers)
{
auto i = std::find_if(children.begin(), children.end(),
[&](const Child & child) { return child.goal2 == goal; });
if (i == children.end()) return;
if (i->inBuildSlot) {
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
}
children.erase(i);
if (wakeSleepers) {
/* Wake up goals waiting for a build slot. */
for (auto & j : wantingToBuild) {
GoalPtr goal = j.lock();
if (goal) wakeUp(goal);
}
wantingToBuild.clear();
}
}
void Worker::waitForBuildSlot(GoalPtr goal)
{
debug("wait for build slot");
if (getNrLocalBuilds() < settings.maxBuildJobs)
wakeUp(goal); /* we can do it right away */
else
addToWeakGoals(wantingToBuild, goal);
}
void Worker::waitForAnyGoal(GoalPtr goal)
{
debug("wait for any goal");
addToWeakGoals(waitingForAnyGoal, goal);
}
void Worker::waitForAWhile(GoalPtr goal)
{
debug("wait for a while");
addToWeakGoals(waitingForAWhile, goal);
}
void Worker::run(const Goals & _topGoals)
{
std::vector<nix::DerivedPath> topPaths;
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs});
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
}
}
/* Call queryMissing() efficiently query substitutes. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
debug("entered goal loop");
while (1) {
checkInterrupt();
// TODO GC interface?
if (auto localStore = dynamic_cast<LocalStore *>(&store))
localStore->autoGC(false);
/* Call every wake goal (in the ordering established by
CompareGoalPtrs). */
while (!awake.empty() && !topGoals.empty()) {
Goals awake2;
for (auto & i : awake) {
GoalPtr goal = i.lock();
if (goal) awake2.insert(goal);
}
awake.clear();
for (auto & goal : awake2) {
checkInterrupt();
goal->work();
if (topGoals.empty()) break; // stuff may have been cancelled
}
}
if (topGoals.empty()) break;
/* Wait for input. */
if (!children.empty() || !waitingForAWhile.empty())
waitForInput();
else {
if (awake.empty() && 0 == settings.maxBuildJobs)
{
if (getMachines().empty())
throw Error("unable to start any build; either increase '--max-jobs' "
"or enable remote builds."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
else
throw Error("unable to start any build; remote machines may not have "
"all required system features."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
}
assert(!awake.empty());
}
}
/* If --keep-going is not set, it's possible that the main goal
exited while some of its subgoals were still active. But if
--keep-going *is* set, then they must all be finished now. */
assert(!settings.keepGoing || awake.empty());
assert(!settings.keepGoing || wantingToBuild.empty());
assert(!settings.keepGoing || children.empty());
}
void Worker::waitForInput()
{
printMsg(lvlVomit, "waiting for children");
/* Process output from the file descriptors attached to the
children, namely log output and output path creation commands.
We also use this to detect child termination: if we get EOF on
the logger pipe of a build, we assume that the builder has
terminated. */
bool useTimeout = false;
long timeout = 0;
auto before = steady_time_point::clock::now();
/* If we're monitoring for silence on stdout/stderr, or if there
is a build timeout, then wait for input until the first
deadline for any child. */
auto nearest = steady_time_point::max(); // nearest deadline
if (settings.minFree.get() != 0)
// Periodicallty wake up to see if we need to run the garbage collector.
nearest = before + std::chrono::seconds(10);
for (auto & i : children) {
if (!i.respectTimeouts) continue;
if (0 != settings.maxSilentTime)
nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
if (0 != settings.buildTimeout)
nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
}
if (nearest != steady_time_point::max()) {
timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
useTimeout = true;
}
/* If we are polling goals that are waiting for a lock, then wake
up after a few seconds at most. */
if (!waitingForAWhile.empty()) {
useTimeout = true;
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
timeout = std::max(1L,
(long) std::chrono::duration_cast<std::chrono::seconds>(
lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
} else lastWokenUp = steady_time_point::min();
if (useTimeout)
vomit("sleeping %d seconds", timeout);
/* Use select() to wait for the input side of any logger pipe to
become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */
std::vector<struct pollfd> pollStatus;
std::map <int, int> fdToPollStatus;
for (auto & i : children) {
for (auto & j : i.fds) {
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
fdToPollStatus[j] = pollStatus.size() - 1;
}
}
if (poll(pollStatus.data(), pollStatus.size(),
useTimeout ? timeout * 1000 : -1) == -1) {
if (errno == EINTR) return;
throw SysError("waiting for input");
}
auto after = steady_time_point::clock::now();
/* Process all available file descriptors. FIXME: this is
O(children * fds). */
decltype(children)::iterator i;
for (auto j = children.begin(); j != children.end(); j = i) {
i = std::next(j);
checkInterrupt();
GoalPtr goal = j->goal.lock();
assert(goal);
set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
ssize_t rd = ::read(k, buffer.data(), buffer.size());
// FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard?
if (rd == 0 || (rd == -1 && errno == EIO)) {
debug("%1%: got EOF", goal->getName());
goal->handleEOF(k);
j->fds.erase(k);
} else if (rd == -1) {
if (errno != EINTR)
throw SysError("%s: read failed", goal->getName());
} else {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), rd);
string data((char *) buffer.data(), rd);
j->lastOutput = after;
goal->handleChildOutput(k, data);
}
}
}
if (goal->exitCode == Goal::ecBusy &&
0 != settings.maxSilentTime &&
j->respectTimeouts &&
after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
{
goal->timedOut(Error(
"%1% timed out after %2% seconds of silence",
goal->getName(), settings.maxSilentTime));
}
else if (goal->exitCode == Goal::ecBusy &&
0 != settings.buildTimeout &&
j->respectTimeouts &&
after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
{
goal->timedOut(Error(
"%1% timed out after %2% seconds",
goal->getName(), settings.buildTimeout));
}
}
if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
lastWokenUp = after;
for (auto & i : waitingForAWhile) {
GoalPtr goal = i.lock();
if (goal) wakeUp(goal);
}
waitingForAWhile.clear();
}
}
unsigned int Worker::exitStatus()
{
/*
* 1100100
* ^^^^
* |||`- timeout
* ||`-- output hash mismatch
* |`--- build failure
* `---- not deterministic
*/
unsigned int mask = 0;
bool buildFailure = permanentFailure || timedOut || hashMismatch;
if (buildFailure)
mask |= 0x04; // 100
if (timedOut)
mask |= 0x01; // 101
if (hashMismatch)
mask |= 0x02; // 102
if (checkMismatch) {
mask |= 0x08; // 104
}
if (mask)
mask |= 0x60;
return mask ? mask : 1;
}
bool Worker::pathContentsGood(const StorePath & path)
{
auto i = pathContentsGoodCache.find(path);
if (i != pathContentsGoodCache.end()) return i->second;
printInfo("checking path '%s'...", store.printStorePath(path));
auto info = store.queryPathInfo(path);
bool res;
if (!pathExists(store.printStorePath(path)))
res = false;
else {
HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
pathContentsGoodCache.insert_or_assign(path, res);
if (!res)
printError("path '%s' is corrupted or missing!", store.printStorePath(path));
return res;
}
void Worker::markContentsGood(const StorePath & path)
{
pathContentsGoodCache.insert_or_assign(path, true);
}
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
return subGoal;
}
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
return subGoal;
}
}

215
src/libstore/build/worker.hh Normal file
View file

@ -0,0 +1,215 @@
#pragma once
#include "types.hh"
#include "lock.hh"
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#include <future>
#include <thread>
namespace nix {
/* Forward definition. */
struct DerivationGoal;
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
/* Workaround for not being able to declare a something like
class PathSubstitutionGoal : public Goal;
even when Goal is a complete type.
This is still a static cast. The purpose of exporting it is to define it in
a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
is opaque. */
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/* A mapping used to remember for each child process to what goal it
belongs, and file descriptors for receiving log data and output
path creation commands. */
struct Child
{
WeakGoalPtr goal;
Goal * goal2; // ugly hackery
set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
steady_time_point timeStarted;
};
/* Forward definition. */
struct HookInstance;
/* The worker class. */
class Worker
{
private:
/* Note: the worker should only have strong pointers to the
top-level goals. */
/* The top-level goals of the worker. */
Goals topGoals;
/* Goals that are ready to do some work. */
WeakGoals awake;
/* Goals waiting for a build slot. */
WeakGoals wantingToBuild;
/* Child processes currently running. */
std::list<Child> children;
/* Number of build slots occupied. This includes local builds and
substitutions but not remote builds via the build hook. */
unsigned int nrLocalBuilds;
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
WeakGoals waitingForAnyGoal;
/* Goals sleeping for a few seconds (polling a lock). */
WeakGoals waitingForAWhile;
/* Last time the goals in `waitingForAWhile' where woken up. */
steady_time_point lastWokenUp;
/* Cache for pathContentsGood(). */
std::map<StorePath, bool> pathContentsGoodCache;
public:
const Activity act;
const Activity actDerivations;
const Activity actSubstitutions;
/* Set if at least one derivation had a BuildError (i.e. permanent
failure). */
bool permanentFailure;
/* Set if at least one derivation had a timeout. */
bool timedOut;
/* Set if at least one derivation fails with a hash mismatch. */
bool hashMismatch;
/* Set if at least one derivation is not deterministic in check mode. */
bool checkMismatch;
Store & store;
std::unique_ptr<HookInstance> hook;
uint64_t expectedBuilds = 0;
uint64_t doneBuilds = 0;
uint64_t failedBuilds = 0;
uint64_t runningBuilds = 0;
uint64_t expectedSubstitutions = 0;
uint64_t doneSubstitutions = 0;
uint64_t failedSubstitutions = 0;
uint64_t runningSubstitutions = 0;
uint64_t expectedDownloadSize = 0;
uint64_t doneDownloadSize = 0;
uint64_t expectedNarSize = 0;
uint64_t doneNarSize = 0;
/* Whether to ask the build hook if it can build a derivation. If
it answers with "decline-permanently", we don't try again. */
bool tryBuildHook = true;
Worker(Store & store);
~Worker();
/* Make a goal (with caching). */
/* derivation goal */
private:
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const StringSet & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
public:
std::shared_ptr<DerivationGoal> makeDerivationGoal(
const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
/* Wake up a goal (i.e., there is something for it to do). */
void wakeUp(GoalPtr goal);
/* Return the number of local build and substitution processes
currently running (but not remote builds via the build
hook). */
unsigned int getNrLocalBuilds();
/* Registers a running child process. `inBuildSlot' means that
the process counts towards the jobs limit. */
void childStarted(GoalPtr goal, const set<int> & fds,
bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be
false if there is no sense in waking up goals that are sleeping
because they can't run yet (e.g., there is no free build slot,
or the hook would still say `postpone'). */
void childTerminated(Goal * goal, bool wakeSleepers = true);
/* Put `goal' to sleep until a build slot becomes available (which
might be right away). */
void waitForBuildSlot(GoalPtr goal);
/* Wait for any goal to finish. Pretty indiscriminate way to
wait for some resource that some other goal is holding. */
void waitForAnyGoal(GoalPtr goal);
/* Wait for a few seconds and then retry this goal. Used when
waiting for a lock held by another process. This kind of
polling is inefficient, but POSIX doesn't really provide a way
to wait for multiple locks in the main select() loop. */
void waitForAWhile(GoalPtr goal);
/* Loop until the specified top-level goals have finished. */
void run(const Goals & topGoals);
/* Wait for input to become available. */
void waitForInput();
unsigned int exitStatus();
/* Check whether the given valid path exists and has the right
contents. */
bool pathContentsGood(const StorePath & path);
void markContentsGood(const StorePath & path);
void updateProgress()
{
actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
}
};
}

10
src/libstore/builtins/buildenv.cc
View file

@ -22,10 +22,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
srcFiles = readDirectory(srcDir);
} catch (SysError & e) {
if (e.errNo == ENOTDIR) {
logWarning({
.name = "Create links - directory",
.hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
});
warn("not including '%s' in the user environment because it's not a directory", srcDir);
return;
}
throw;
@ -44,10 +41,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
throw SysError("getting status of '%1%'", srcFile);
} catch (SysError & e) {
if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
logWarning({
.name = "Create links - skipping symlink",
.hint = hintfmt("skipping dangling symlink '%s'", dstFile)
});
warn("skipping dangling symlink '%s'", dstFile);
continue;
}
throw;

14
src/libstore/builtins/fetchurl.cc
View file

@ -58,6 +58,20 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
/* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat")
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
return;
} catch (Error & e) {
debug(e.what());
}
/* Otherwise try the specified URL. */
fetch(mainUrl);
}

12
src/libstore/ca-specific-schema.sql Normal file
View file

@ -0,0 +1,12 @@
-- Extension of the sql schema for content-addressed derivations.
-- Won't be loaded unless the experimental feature `ca-derivations`
-- is enabled
create table if not exists Realisations (
drvPath text not null,
outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null,
signatures text, -- space-separated list
primary key (drvPath, outputName),
foreign key (outputPath) references ValidPaths(id) on delete cascade
);

84
src/libstore/content-address.cc
View file

@ -4,11 +4,13 @@
namespace nix {
std::string FixedOutputHash::printMethodAlgo() const {
std::string FixedOutputHash::printMethodAlgo() const
{
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
std::string makeFileIngestionPrefix(const FileIngestionMethod m)
{
switch (m) {
case FileIngestionMethod::Flat:
return "";
@ -26,7 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+ hash.to_string(Base32, true);
}
std::string renderContentAddress(ContentAddress ca) {
std::string renderContentAddress(ContentAddress ca)
{
return std::visit(overloaded {
[](TextHash th) {
return "text:" + th.hash.to_string(Base32, true);
@ -37,54 +40,97 @@ std::string renderContentAddress(ContentAddress ca) {
}, ca);
}
ContentAddress parseContentAddress(std::string_view rawCa) {
auto rest = rawCa;
std::string renderContentAddressMethod(ContentAddressMethod cam)
{
return std::visit(overloaded {
[](TextHashMethod &th) {
return std::string{"text:"} + printHashType(htSHA256);
},
[](FixedOutputHashMethod &fshm) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
}
}, cam);
}
/*
Parses content address strings up to the hash.
*/
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
{
std::string_view wholeInput { rest };
std::string_view prefix;
{
auto optPrefix = splitPrefixTo(rest, ':');
if (!optPrefix)
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", rawCa);
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", wholeInput);
prefix = *optPrefix;
}
auto parseHashType_ = [&](){
auto hashTypeRaw = splitPrefixTo(rest, ':');
if (!hashTypeRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", rawCa);
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
HashType hashType = parseHashType(*hashTypeRaw);
return std::move(hashType);
};
// Switch on prefix
if (prefix == "text") {
// No parsing of the method, "text" only support flat.
// No parsing of the ingestion method, "text" only support flat.
HashType hashType = parseHashType_();
if (hashType != htSHA256)
throw Error("text content address hash should use %s, but instead uses %s",
printHashType(htSHA256), printHashType(hashType));
return TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
};
return TextHashMethod {};
} else if (prefix == "fixed") {
// Parse method
auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_();
return FixedOutputHash {
.method = method,
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
return FixedOutputHashMethod {
.fileIngestionMethod = method,
.hashType = std::move(hashType),
};
} else
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
}
ContentAddress parseContentAddress(std::string_view rawCa) {
auto rest = rawCa;
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
return std::visit(
overloaded {
[&](TextHashMethod thm) {
return ContentAddress(TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
});
},
[&](FixedOutputHashMethod fohMethod) {
return ContentAddress(FixedOutputHash {
.method = fohMethod.fileIngestionMethod,
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
});
},
}, caMethod);
}
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
{
std::string_view asPrefix {std::string{caMethod} + ":"};
return parseContentAddressMethodPrefix(asPrefix);
}
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)
{
return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt);
};
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
};
std::string renderContentAddress(std::optional<ContentAddress> ca) {
std::string renderContentAddress(std::optional<ContentAddress> ca)
{
return ca ? renderContentAddress(*ca) : "";
}

19
src/libstore/content-address.hh
View file

@ -55,4 +55,23 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
/*
We only have one way to hash text with references, so this is single-value
type is only useful in std::variant.
*/
struct TextHashMethod { };
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
};
typedef std::variant<
TextHashMethod,
FixedOutputHashMethod
> ContentAddressMethod;
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
std::string renderContentAddressMethod(ContentAddressMethod caMethod);
}

54
src/libstore/crypto.cc
View file

@ -2,21 +2,19 @@
#include "util.hh"
#include "globals.hh"
#if HAVE_SODIUM
#include <sodium.h>
#endif
namespace nix {
static std::pair<std::string, std::string> split(const string & s)
static std::pair<std::string_view, std::string_view> split(std::string_view s)
{
size_t colon = s.find(':');
if (colon == std::string::npos || colon == 0)
return {"", ""};
return {std::string(s, 0, colon), std::string(s, colon + 1)};
return {s.substr(0, colon), s.substr(colon + 1)};
}
Key::Key(const string & s)
Key::Key(std::string_view s)
{
auto ss = split(s);
@ -29,62 +27,57 @@ Key::Key(const string & s)
key = base64Decode(key);
}
SecretKey::SecretKey(const string & s)
std::string Key::to_string() const
{
return name + ":" + base64Encode(key);
}
SecretKey::SecretKey(std::string_view s)
: Key(s)
{
#if HAVE_SODIUM
if (key.size() != crypto_sign_SECRETKEYBYTES)
throw Error("secret key is not valid");
#endif
}
#if !HAVE_SODIUM
[[noreturn]] static void noSodium()
std::string SecretKey::signDetached(std::string_view data) const
{
throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
}
#endif
std::string SecretKey::signDetached(const std::string & data) const
{
#if HAVE_SODIUM
unsigned char sig[crypto_sign_BYTES];
unsigned long long sigLen;
crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(),
(unsigned char *) key.data());
return name + ":" + base64Encode(std::string((char *) sig, sigLen));
#else
noSodium();
#endif
}
PublicKey SecretKey::toPublicKey() const
{
#if HAVE_SODIUM
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data());
return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES));
#else
noSodium();
#endif
}
PublicKey::PublicKey(const string & s)
SecretKey SecretKey::generate(std::string_view name)
{
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
unsigned char sk[crypto_sign_SECRETKEYBYTES];
if (crypto_sign_keypair(pk, sk) != 0)
throw Error("key generation failed");
return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES));
}
PublicKey::PublicKey(std::string_view s)
: Key(s)
{
#if HAVE_SODIUM
if (key.size() != crypto_sign_PUBLICKEYBYTES)
throw Error("public key is not valid");
#endif
}
bool verifyDetached(const std::string & data, const std::string & sig,
const PublicKeys & publicKeys)
{
#if HAVE_SODIUM
auto ss = split(sig);
auto key = publicKeys.find(ss.first);
auto key = publicKeys.find(std::string(ss.first));
if (key == publicKeys.end()) return false;
auto sig2 = base64Decode(ss.second);
@ -94,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig,
return crypto_sign_verify_detached((unsigned char *) sig2.data(),
(unsigned char *) data.data(), data.size(),
(unsigned char *) key->second.key.data()) == 0;
#else
noSodium();
#endif
}
PublicKeys getDefaultPublicKeys()

24
src/libstore/crypto.hh
View file

@ -13,32 +13,40 @@ struct Key
/* Construct Key from a string in the format
‘<name>:<key-in-base64>’. */
Key(const std::string & s);
Key(std::string_view s);
std::string to_string() const;
protected:
Key(const std::string & name, const std::string & key)
: name(name), key(key) { }
Key(std::string_view name, std::string && key)
: name(name), key(std::move(key)) { }
};
struct PublicKey;
struct SecretKey : Key
{
SecretKey(const std::string & s);
SecretKey(std::string_view s);
/* Return a detached signature of the given string. */
std::string signDetached(const std::string & s) const;
std::string signDetached(std::string_view s) const;
PublicKey toPublicKey() const;
static SecretKey generate(std::string_view name);
private:
SecretKey(std::string_view name, std::string && key)
: Key(name, std::move(key)) { }
};
struct PublicKey : Key
{
PublicKey(const std::string & data);
PublicKey(std::string_view data);
private:
PublicKey(const std::string & name, const std::string & key)
: Key(name, key) { }
PublicKey(std::string_view name, std::string && key)
: Key(name, std::move(key)) { }
friend struct SecretKey;
};

397
src/libstore/daemon.cc
View file

@ -2,7 +2,7 @@
#include "monitor-fd.hh"
#include "worker-protocol.hh"
#include "store-api.hh"
#include "local-store.hh"
#include "path-with-outputs.hh"
#include "finally.hh"
#include "affinity.hh"
#include "archive.hh"
@ -102,17 +102,20 @@ struct TunnelLogger : public Logger
/* stopWork() means that we're done; stop sending stderr to the
client. */
void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
void stopWork(const Error * ex = nullptr)
{
auto state(state_.lock());
state->canSendStderr = false;
if (success)
if (!ex)
to << STDERR_LAST;
else {
to << STDERR_ERROR << msg;
if (status != 0) to << status;
if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
to << STDERR_ERROR << *ex;
} else {
to << STDERR_ERROR << ex->what() << ex->status;
}
}
}
@ -151,10 +154,10 @@ struct TunnelSink : Sink
{
Sink & to;
TunnelSink(Sink & to) : to(to) { }
virtual void operator () (const unsigned char * data, size_t len)
void operator () (std::string_view data)
{
to << STDERR_WRITE;
writeString(data, len, to);
writeString(data, to);
}
};
@ -163,7 +166,7 @@ struct TunnelSource : BufferedSource
Source & from;
BufferedSink & to;
TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { }
size_t readUnbuffered(unsigned char * data, size_t len) override
size_t readUnbuffered(char * data, size_t len) override
{
to << STDERR_READ << len;
to.flush();
@ -213,6 +216,8 @@ struct ClientSettings
for (auto & s : ss)
if (trusted.count(s))
subs.push_back(s);
else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
subs.push_back(s + "/");
else
warn("ignoring untrusted substituter '%s'", s);
res = subs;
@ -229,10 +234,8 @@ struct ClientSettings
settings.set(name, value);
else if (setSubstituters(settings.substituters))
;
else if (setSubstituters(settings.extraSubstituters))
;
else
warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
} catch (UsageError & e) {
warn(e.what());
}
@ -240,6 +243,35 @@ struct ClientSettings
}
};
static void writeValidPathInfo(
ref<Store> store,
unsigned int clientVersion,
Sink & to,
std::shared_ptr<const ValidPathInfo> info)
{
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash.to_string(Base16, false);
worker_proto::write(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
<< renderContentAddress(info->ca);
}
}
static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from)
{
std::vector<DerivedPath> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
} else {
for (auto & s : readStrings<Strings>(from))
reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
}
return reqs;
}
static void performOp(TunnelLogger * logger, ref<Store> store,
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
Source & from, BufferedSink & to, unsigned int op)
@ -256,11 +288,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryValidPaths: {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
SubstituteFlag substitute = NoSubstitute;
if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
substitute = readInt(from) ? Substitute : NoSubstitute;
}
logger->startWork();
auto res = store->queryValidPaths(paths);
if (substitute) {
store->substitutePaths(paths);
}
auto res = store->queryValidPaths(paths, substitute);
logger->stopWork();
writeStorePaths(*store, to, res);
worker_proto::write(*store, to, res);
break;
}
@ -276,11 +317,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePaths: {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
logger->startWork();
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
writeStorePaths(*store, to, res);
worker_proto::write(*store, to, res);
break;
}
@ -289,7 +330,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
to << hash->to_string(Base16, false);
to << hash.to_string(Base16, false);
break;
}
@ -309,7 +350,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
paths = store->queryValidDerivers(path);
else paths = store->queryDerivationOutputs(path);
logger->stopWork();
writeStorePaths(*store, to, paths);
worker_proto::write(*store, to, paths);
break;
}
@ -325,9 +366,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryDerivationOutputMap: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
OutputPathMap outputs = store->queryDerivationOutputMap(path);
auto outputs = store->queryPartialDerivationOutputMap(path);
logger->stopWork();
writeOutputPathMap(*store, to, outputs);
worker_proto::write(*store, to, outputs);
break;
}
@ -350,54 +391,90 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopAddToStore: {
HashType hashAlgo;
std::string baseName;
FileIngestionMethod method;
{
bool fixed;
uint8_t recursive;
std::string hashAlgoRaw;
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
hashAlgoRaw = "sha256";
method = FileIngestionMethod::Recursive;
if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
auto name = readString(from);
auto camStr = readString(from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
bool repairBool;
from >> repairBool;
auto repair = RepairFlag{repairBool};
logger->startWork();
auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork();
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded {
[&](TextHashMethod &_) {
// We could stream this by changing Store
std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path);
},
[&](FixedOutputHashMethod &fohm) {
if (!refs.empty())
throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
return store->queryPathInfo(path);
},
}, contentAddressMethod);
}();
logger->stopWork();
to << store->printStorePath(pathInfo->path);
writeValidPathInfo(store, clientVersion, to, pathInfo);
} else {
HashType hashAlgo;
std::string baseName;
FileIngestionMethod method;
{
bool fixed;
uint8_t recursive;
std::string hashAlgoRaw;
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
hashAlgoRaw = "sha256";
method = FileIngestionMethod::Recursive;
}
hashAlgo = parseHashType(hashAlgoRaw);
}
hashAlgo = parseHashType(hashAlgoRaw);
StringSink saved;
TeeSource savedNARSource(from, saved);
RetrieveRegularNARSink savedRegular { saved };
if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNARSource);
} else
parseDump(savedRegular, from);
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
// FIXME: try to stream directly from `from`.
StringSource dumpSource { *saved.s };
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
logger->stopWork();
to << store->printStorePath(path);
}
StringSink saved;
TeeSource savedNARSource(from, saved);
RetrieveRegularNARSink savedRegular { saved };
if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNARSource);
} else
parseDump(savedRegular, from);
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
// FIXME: try to stream directly from `from`.
StringSource dumpSource { *saved.s };
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
logger->stopWork();
to << store->printStorePath(path);
break;
}
case wopAddTextToStore: {
string suffix = readString(from);
string s = readString(from);
auto refs = readStorePaths<StorePathSet>(*store, from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
logger->stopWork();
@ -429,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopBuildPaths: {
std::vector<StorePathWithOutputs> drvs;
for (auto & s : readStrings<Strings>(from))
drvs.push_back(store->parsePathWithOutputs(s));
auto drvs = readDerivedPaths(*store, clientVersion, from);
BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from);
@ -454,11 +529,69 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
if (!trusted)
throw Error("you are not privileged to build derivations");
/* Content-addressed derivations are trustless because their output paths
are verified by their content alone, so any derivation is free to
try to produce such a path.
Input-addressed derivation output paths, however, are calculated
from the derivation closure that produced them---even knowing the
root derivation is not enough. That the output data actually came
from those derivations is fundamentally unverifiable, but the daemon
trusts itself on that matter. The question instead is whether the
submitted plan has rights to the output paths it wants to fill, and
at least the derivation closure proves that.
It would have been nice if input-address algorithm merely depended
on the build time closure, rather than depending on the derivation
closure. That would mean input-addressed paths used at build time
would just be trusted and not need their own evidence. This is in
fact fine as the same guarantees would hold *inductively*: either
the remote builder has those paths and already trusts them, or it
needs to build them too and thus their evidence must be provided in
turn. The advantage of this variant algorithm is that the evidence
for input-addressed paths which the remote builder already has
doesn't need to be sent again.
That said, now that we have floating CA derivations, it is better
that people just migrate to those which also solve this problem, and
others. It's the same migration difficulty with strictly more
benefit.
Lastly, do note that when we parse fixed-output content-addressed
derivations, we throw out the precomputed output paths and just
store the hashes, so there aren't two competing sources of truth an
attacker could exploit. */
if (drv.type() == DerivationType::InputAddressed && !trusted)
throw Error("you are not privileged to build input-addressed derivations");
/* Make sure that the non-input-addressed derivations that got this far
are in fact content-addressed if we don't trust them. */
assert(derivationIsCA(drv.type()) || trusted);
/* Recompute the derivation path when we cannot trust the original. */
if (!trusted) {
/* Recomputing the derivation path for input-address derivations
makes it harder to audit them after the fact, since we need the
original not-necessarily-resolved derivation to verify the drv
derivation as adequate claim to the input-addressed output
paths. */
assert(derivationIsCA(drv.type()));
Derivation drv2;
static_cast<BasicDerivation &>(drv2) = drv;
drvPath = writeDerivation(*store, Derivation { drv2 });
}
auto res = store->buildDerivation(drvPath, drv, buildMode);
logger->stopWork();
to << res.status << res.errorMsg;
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
}
if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
worker_proto::write(*store, to, res.builtOutputs);
}
break;
}
@ -518,7 +651,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopCollectGarbage: {
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
from >> options.ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
@ -587,7 +720,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
else {
to << 1
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
writeStorePaths(*store, to, i->second.references);
worker_proto::write(*store, to, i->second.references);
to << i->second.downloadSize
<< i->second.narSize;
}
@ -598,11 +731,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
SubstitutablePathInfos infos;
StorePathCAMap pathsMap = {};
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
} else
pathsMap = readStorePathCAMap(*store, from);
pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
logger->startWork();
store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
@ -610,7 +743,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
for (auto & i : infos) {
to << store->printStorePath(i.first)
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
writeStorePaths(*store, to, i.second.references);
worker_proto::write(*store, to, i.second.references);
to << i.second.downloadSize << i.second.narSize;
}
break;
@ -620,7 +753,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto paths = store->queryAllValidPaths();
logger->stopWork();
writeStorePaths(*store, to, paths);
worker_proto::write(*store, to, paths);
break;
}
@ -637,15 +770,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (info) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash->to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
<< renderContentAddress(info->ca);
}
writeValidPathInfo(store, clientVersion, to, info);
} else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
to << 0;
@ -688,18 +813,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
logger->stopWork();
dumpPath(store->printStorePath(path), to);
dumpPath(store->toRealPath(path), to);
break;
}
case wopAddToStoreNar: {
bool repair, dontCheckSigs;
ValidPathInfo info(store->parseStorePath(readString(from)));
auto path = store->parseStorePath(readString(from));
auto deriver = readString(from);
auto narHash = Hash::parseAny(readString(from), htSHA256);
ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.narHash = Hash::parseAny(readString(from), htSHA256);
info.references = readStorePaths<StorePathSet>(*store, from);
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
info.ca = parseContentAddressOpt(readString(from));
@ -710,59 +836,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.ultimate = false;
if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
struct FramedSource : Source
{
Source & from;
bool eof = false;
std::vector<unsigned char> pending;
size_t pos = 0;
FramedSource(Source & from) : from(from)
{ }
~FramedSource()
{
if (!eof) {
while (true) {
auto n = readInt(from);
if (!n) break;
std::vector<unsigned char> data(n);
from(data.data(), n);
}
}
}
size_t read(unsigned char * data, size_t len) override
{
if (eof) throw EndOfFile("reached end of FramedSource");
if (pos >= pending.size()) {
size_t len = readInt(from);
if (!len) {
eof = true;
return 0;
}
pending = std::vector<unsigned char>(len);
pos = 0;
from(pending.data(), len);
}
auto n = std::min(len, pending.size() - pos);
memcpy(data, pending.data() + pos, n);
pos += n;
return n;
}
};
logger->startWork();
{
FramedSource source(from);
store->addToStore(info, source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs);
}
logger->stopWork();
}
@ -791,21 +870,40 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryMissing: {
std::vector<StorePathWithOutputs> targets;
for (auto & s : readStrings<Strings>(from))
targets.push_back(store->parsePathWithOutputs(s));
auto targets = readDerivedPaths(*store, clientVersion, from);
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
writeStorePaths(*store, to, willBuild);
writeStorePaths(*store, to, willSubstitute);
writeStorePaths(*store, to, unknown);
worker_proto::write(*store, to, willBuild);
worker_proto::write(*store, to, willSubstitute);
worker_proto::write(*store, to, unknown);
to << downloadSize << narSize;
break;
}
case wopRegisterDrvOutput: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto outputPath = StorePath(readString(from));
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
logger->stopWork();
break;
}
case wopQueryRealisation: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto info = store->queryRealisation(outputId);
logger->stopWork();
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
break;
}
default:
throw Error("invalid operation %1%", op);
}
@ -817,8 +915,7 @@ void processConnection(
FdSink & to,
TrustedFlag trusted,
RecursiveFlag recursive,
const std::string & userName,
uid_t userId)
std::function<void(Store &)> authHook)
{
auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
@ -859,15 +956,7 @@ void processConnection(
/* If we can't accept clientVersion, then throw an error
*here* (not above). */
#if 0
/* Prevent users from doing something very dangerous. */
if (geteuid() == 0 &&
querySetting("build-users-group", "") == "")
throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
#endif
store->createUser(userName, userId);
authHook(*store);
tunnelLogger->stopWork();
to.flush();
@ -894,10 +983,11 @@ void processConnection(
during addTextToStore() / importPath(). If that
happens, just send the error message and exit. */
bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
tunnelLogger->stopWork(false, e.msg(), e.status);
tunnelLogger->stopWork(&e);
if (!errorAllowed) throw;
} catch (std::bad_alloc & e) {
tunnelLogger->stopWork(false, "Nix daemon out of memory", 1);
auto ex = Error("Nix daemon out of memory");
tunnelLogger->stopWork(&ex);
throw;
}
@ -906,8 +996,13 @@ void processConnection(
assert(!tunnelLogger->state_.lock()->canSendStderr);
};
} catch (Error & e) {
tunnelLogger->stopWork(&e);
to.flush();
return;
} catch (std::exception & e) {
tunnelLogger->stopWork(false, e.what(), 1);
auto ex = Error(e.what());
tunnelLogger->stopWork(&ex);
to.flush();
return;
}

9
src/libstore/daemon.hh
View file

@ -1,3 +1,5 @@
#pragma once
#include "serialise.hh"
#include "store-api.hh"
@ -12,7 +14,10 @@ void processConnection(
FdSink & to,
TrustedFlag trusted,
RecursiveFlag recursive,
const std::string & userName,
uid_t userId);
/* Arbitrary hook to check authorization / initialize user data / whatever
after the protocol has been negotiated. The idea is that this function
and everything it calls doesn't know about this stuff, and the
`nix-daemon` handles that instead. */
std::function<void(Store &)> authHook);
}

408
src/libstore/derivations.cc
View file

@ -7,7 +7,7 @@
namespace nix {
std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
@ -15,21 +15,32 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
},
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return {
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
dof.path(store, drvName, outputName)
};
},
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
return std::nullopt;
},
[](DerivationOutputDeferred) -> std::optional<StorePath> {
return std::nullopt;
},
}, output);
}
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool derivationIsCA(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return true;
case DerivationType::DeferredInputAddressed: return false;
};
// Since enums can have non-variant values, but making a `default:` would
// disable exhaustiveness warnings.
@ -41,15 +52,28 @@ bool derivationIsFixed(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
bool derivationHasKnownOutputPaths(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return true;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
bool derivationIsImpure(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
@ -61,8 +85,8 @@ bool BasicDerivation::isBuiltin() const
}
StorePath writeDerivation(ref<Store> store,
const Derivation & drv, std::string_view name, RepairFlag repair)
StorePath writeDerivation(Store & store,
const Derivation & drv, RepairFlag repair, bool readOnly)
{
auto references = drv.inputSrcs;
for (auto & i : drv.inputDrvs)
@ -70,11 +94,11 @@ StorePath writeDerivation(ref<Store> store,
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
auto suffix = std::string(name) + drvExtension;
auto contents = drv.unparse(*store, false);
return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references, repair);
auto suffix = std::string(drv.name) + drvExtension;
auto contents = drv.unparse(store, false);
return readOnly || settings.readOnlyMode
? store.computeStorePathForText(suffix, contents, references)
: store.addTextToStore(suffix, contents, references, repair);
}
@ -106,12 +130,15 @@ static string parseString(std::istream & str)
return res;
}
static void validatePath(std::string_view s) {
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
}
static Path parsePath(std::istream & str)
{
string s = parseString(str);
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
auto s = parseString(str);
validatePath(s);
return s;
}
@ -139,47 +166,63 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
static DerivationOutput parseDerivationOutput(const Store & store,
std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
hashAlgo = hashAlgo.substr(2);
}
const HashType hashType = parseHashType(hashAlgo);
return hash != ""
? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else
const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
},
};
} else {
settings.requireExperimentalFeature("ca-derivations");
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
};
}
} else {
if (pathS == "") {
return DerivationOutput {
.output = DerivationOutputDeferred { }
};
}
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path),
.path = store.parseStorePath(pathS),
}
};
}
}
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{
expect(str, ","); const auto pathS = parseString(str);
expect(str, ","); const auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
return parseDerivationOutput(store, pathS, hashAlgo, hash);
}
static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{
Derivation drv;
drv.name = name;
@ -227,34 +270,6 @@ static Derivation parseDerivation(const Store & store, std::string && s, std::st
}
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
{
try {
return parseDerivation(store, readFile(drvPath), name);
} catch (FormatError & e) {
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
}
}
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
ensurePath(drvPath);
return readDerivation(drvPath);
}
Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor();
try {
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
}
}
static void printString(string & res, std::string_view s)
{
char buf[s.size() * 2 + 2];
@ -316,20 +331,27 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputDeferred) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
}, i.second.output);
s += ')';
}
@ -382,9 +404,19 @@ bool isDerivation(const string & fileName)
}
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
std::string res { drvName };
if (outputName != "out") {
res += "-";
res += outputName;
}
return res;
}
DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
@ -403,29 +435,34 @@ DerivationType BasicDerivation::type() const
throw Error("All floating outputs must use the same hash type");
}
},
[&](DerivationOutputDeferred _) {
deferredIAOutputs.insert(i.first);
},
}, i.second.output);
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::InputAddressed;
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
return DerivationType::CAFixed;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::CAFloating;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
return DerivationType::DeferredInputAddressed;
} else {
throw Error("Can't mix derivation output types");
}
}
DrvHashes drvHashes;
Sync<DrvHashes> drvHashes;
/* pathDerivationModulo and hashDerivationModulo are mutually recursive
*/
@ -433,20 +470,22 @@ DrvHashes drvHashes;
/* Look up the derivation by value and memoize the
`hashDerivationModulo` call.
*/
static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
{
auto h = drvHashes.find(drvPath);
if (h == drvHashes.end()) {
assert(store.isValidPath(drvPath));
// Cache it
h = drvHashes.insert_or_assign(
drvPath,
hashDerivationModulo(
store,
store.readDerivation(drvPath),
false)).first;
{
auto hashes = drvHashes.lock();
auto h = hashes->find(drvPath);
if (h != hashes->end()) {
return h->second;
}
}
return h->second;
auto h = hashDerivationModulo(
store,
store.readInvalidDerivation(drvPath),
false);
// Cache it
drvHashes.lock()->insert_or_assign(drvPath, h);
return h;
}
/* See the header for interface details. These are the implementation details.
@ -468,10 +507,9 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
bool isDeferred = false;
/* Return a fixed hash for fixed-output derivations. */
switch (drv.type()) {
case DerivationType::CAFloating:
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
@ -479,13 +517,18 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.path(store, drv.name)));
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return outputHashes;
}
case DerivationType::CAFloating:
isDeferred = true;
break;
case DerivationType::InputAddressed:
break;
case DerivationType::DeferredInputAddressed:
break;
}
/* For other derivations, replace the inputs paths with recursive
@ -498,6 +541,10 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
[&](Hash drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
},
[&](DeferredHash deferredHash) {
isDeferred = true;
inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
},
// CA derivation's output hashes
[&](CaOutputHashes outputHashes) {
std::set<std::string> justOut = { "out" };
@ -512,15 +559,34 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}, res);
}
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
if (isDeferred)
return DeferredHash { hash };
else
return hash;
}
std::string StorePathWithOutputs::to_string(const Store & store) const
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv)
{
return outputs.empty()
? store.printStorePath(path)
: store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
std::map<std::string, Hash> res;
std::visit(overloaded {
[&](Hash drvHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, drvHash});
}
},
[&](DeferredHash deferredHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, deferredHash.hash});
}
},
[&](CaOutputHashes outputHashes) {
res = outputHashes;
},
}, hashDerivationModulo(store, drv, true));
return res;
}
@ -530,49 +596,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
}
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputs)
paths.insert(i.second.path(store, name));
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
const auto pathS = readString(in);
const auto hashAlgo = readString(in);
const auto hash = readString(in);
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
auto hashType = parseHashType(hashAlgo);
return hash != ""
? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path),
}
};
return parseDerivationOutput(store, pathS, hashAlgo, hash);
}
StringSet BasicDerivation::outputNames() const
@ -583,6 +613,16 @@ StringSet BasicDerivation::outputNames() const
return names;
}
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair(
output.first,
std::make_pair(output.second, output.second.path(store, name, output.first))
)
);
return outsAndOptPaths;
}
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
auto nameWithSuffix = drvPath.name();
@ -605,7 +645,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);
@ -624,23 +664,31 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
{
out << drv.outputs.size();
for (auto & i : drv.outputs) {
out << i.first
<< store.printStorePath(i.second.path(store, drv.name));
out << i.first;
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
out << "" << "";
out << store.printStorePath(doi.path)
<< ""
<< "";
},
[&](DerivationOutputCAFixed dof) {
out << dof.hash.printMethodAlgo()
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false);
},
[&](DerivationOutputCAFloating dof) {
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
[&](DerivationOutputDeferred) {
out << ""
<< ""
<< "";
},
}, i.second.output);
}
writeStorePaths(store, out, drv.inputSrcs);
worker_proto::write(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)
@ -654,5 +702,81 @@ std::string hashPlaceholder(const std::string & outputName)
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
}
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
}
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
debug("Rewriting the derivation");
for (auto &rewrite: rewrites) {
debug("rewriting %s as %s", rewrite.first, rewrite.second);
}
drv.builder = rewriteStrings(drv.builder, rewrites);
for (auto & arg: drv.args) {
arg = rewriteStrings(arg, rewrites);
}
StringPairs newEnv;
for (auto & envVar: drv.env) {
auto envName = rewriteStrings(envVar.first, rewrites);
auto envValue = rewriteStrings(envVar.second, rewrites);
newEnv.emplace(envName, envValue);
}
drv.env = newEnv;
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
Hash h = std::get<Hash>(hashModulo);
auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(outPath),
},
};
}
}
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
for (auto & input : inputDrvs) {
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
StringSet newOutputNames;
for (auto & outputName : input.second) {
auto actualPathOpt = inputDrvOutputs.at(outputName);
if (!actualPathOpt) {
warn("output %s of input %s missing, aborting the resolving",
outputName,
store.printStorePath(input.first)
);
return std::nullopt;
}
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, input.first, outputName),
store.printStorePath(actualPath));
resolved.inputSrcs.insert(std::move(actualPath));
}
}
rewriteDerivation(store, resolved, inputRewrites);
return resolved;
}
}

104
src/libstore/derivations.hh
View file

@ -4,6 +4,7 @@
#include "types.hh"
#include "hash.hh"
#include "content-address.hh"
#include "sync.hh"
#include <map>
#include <variant>
@ -17,8 +18,6 @@ namespace nix {
/* The traditional non-fixed-output derivation type. */
struct DerivationOutputInputAddressed
{
/* Will need to become `std::optional<StorePath>` once input-addressed
derivations are allowed to depend on cont-addressed derivations */
StorePath path;
};
@ -27,6 +26,7 @@ struct DerivationOutputInputAddressed
struct DerivationOutputCAFixed
{
FixedOutputHash hash; /* hash used for expected hash computation */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
};
/* Floating-output derivations, whose output paths are content addressed, but
@ -39,25 +39,35 @@ struct DerivationOutputCAFloating
HashType hashType;
};
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
* known atm
*/
struct DerivationOutputDeferred {};
struct DerivationOutput
{
std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating
DerivationOutputCAFloating,
DerivationOutputDeferred
> output;
std::optional<HashType> hashAlgoOpt(const Store & store) const;
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
/* DEPRECATED: Remove after CA drvs are fully implemented */
StorePath path(const Store & store, std::string_view drvName) const {
auto p = pathOpt(store, drvName);
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
return *p;
}
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
interface provided by BasicDerivation::outputsAndOptPaths */
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
/* These are analogues to the previous DerivationOutputs data type, but they
also contains, for each output, the (optional) store path in which it would
be written. To calculate values of these types, see the corresponding
functions in BasicDerivation */
typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>>
DerivationOutputsAndOptPaths;
/* For inputs that are sub-derivations, we specify exactly which
output IDs we are interested in. */
typedef std::map<StorePath, StringSet> DerivationInputs;
@ -66,6 +76,7 @@ typedef std::map<string, string> StringPairs;
enum struct DerivationType : uint8_t {
InputAddressed,
DeferredInputAddressed,
CAFixed,
CAFloating,
};
@ -83,6 +94,11 @@ bool derivationIsFixed(DerivationType);
derivation is controlled separately. Never true for non-CA derivations. */
bool derivationIsImpure(DerivationType);
/* Does the derivation knows its own output paths?
* Only true when there's no floating-ca derivation involved in the closure.
*/
bool derivationHasKnownOutputPaths(DerivationType);
struct BasicDerivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
@ -93,7 +109,7 @@ struct BasicDerivation
StringPairs env;
std::string name;
BasicDerivation() { }
BasicDerivation() = default;
virtual ~BasicDerivation() { };
bool isBuiltin() const;
@ -101,12 +117,14 @@ struct BasicDerivation
/* Return true iff this is a fixed-output derivation. */
DerivationType type() const;
/* Return the output paths of a derivation. */
StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
/* Calculates the maps that contains all the DerivationOutputs, but
augmented with knowledge of the Store paths they would be written
into. */
DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
static std::string_view nameFromPath(const StorePath & storePath);
};
@ -118,7 +136,17 @@ struct Derivation : BasicDerivation
std::string unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs = nullptr) const;
Derivation() { }
/* Return the underlying basic derivation but with these changes:
1. Input drvs are emptied, but the outputs of them that were used are
added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
};
@ -127,22 +155,34 @@ class Store;
enum RepairFlag : bool { NoRepair = false, Repair = true };
/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store,
const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
StorePath writeDerivation(Store & store,
const Derivation & drv,
RepairFlag repair = NoRepair,
bool readOnly = false);
/* Read a derivation from a file. */
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
// FIXME: remove
bool isDerivation(const string & fileName);
/* Calculate the name that will be used for the store path for this
output.
This is usually <drv-name>-<output-name>, but is just <drv-name> when
the output name is "out". */
std::string outputPathName(std::string_view drvName, std::string_view outputName);
// known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
struct DeferredHash { Hash hash; };
typedef std::variant<
Hash, // regular DRV normalized hash
CaOutputHashes
CaOutputHashes, // Fixed-output derivation hashes
DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
> DrvHashModulo;
/* Returns hashes with the details of fixed-output subderivations
@ -170,10 +210,17 @@ typedef std::variant<
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/*
Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references).
*/
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe
// FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes;
bool wantOutput(const string & output, const std::set<string> & wanted);
@ -183,6 +230,21 @@ struct Sink;
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
/* This creates an opaque and almost certainly unique string
deterministically from the output name.
It is used as a placeholder to allow derivations to refer to their
own outputs without needing to use the hash of a derivation in
itself, making the hash near-impossible to calculate. */
std::string hashPlaceholder(const std::string & outputName);
/* This creates an opaque and almost certainly unique string
deterministically from a derivation path and output name.
It is used as a placeholder to allow derivations to refer to
content-addressed paths whose content --- and thus the path
themselves --- isn't yet known. This occurs when a derivation has a
dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
}

77
src/libstore/derived-path.cc Normal file
View file

@ -0,0 +1,77 @@
#include "derived-path.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix {
nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
nlohmann::json res;
res["path"] = store->printStorePath(path);
return res;
}
nlohmann::json DerivedPathWithHints::Built::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = path ? store->printStorePath(*path) : "";
}
return res;
}
nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store) {
auto res = nlohmann::json::array();
for (const DerivedPathWithHints & buildable : buildables) {
std::visit([&res, store](const auto & buildable) {
res.push_back(buildable.toJSON(store));
}, buildable.raw());
}
return res;
}
std::string DerivedPath::Opaque::to_string(const Store & store) const {
return store.printStorePath(path);
}
std::string DerivedPath::Built::to_string(const Store & store) const {
return store.printStorePath(drvPath)
+ "!"
+ (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
}
std::string DerivedPath::to_string(const Store & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
this->raw());
}
DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s)
{
return {store.parseStorePath(s)};
}
DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s)
{
size_t n = s.find("!");
assert(n != s.npos);
auto drvPath = store.parseStorePath(s.substr(0, n));
auto outputsS = s.substr(n + 1);
std::set<string> outputs;
if (outputsS != "*")
outputs = tokenizeString<std::set<string>>(outputsS);
return {drvPath, outputs};
}
DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? (DerivedPath) DerivedPath::Opaque::parse(store, s)
: (DerivedPath) DerivedPath::Built::parse(store, s);
}
}

129
src/libstore/derived-path.hh Normal file
View file

@ -0,0 +1,129 @@
#pragma once
#include "util.hh"
#include "path.hh"
#include <optional>
#include <nlohmann/json_fwd.hpp>
namespace nix {
class Store;
/**
* An opaque derived path.
*
* Opaque derived paths are just store paths, and fully evaluated. They
* cannot be simplified further. Since they are opaque, they cannot be
* built, but they can fetched.
*/
struct DerivedPathOpaque {
StorePath path;
nlohmann::json toJSON(ref<Store> store) const;
std::string to_string(const Store & store) const;
static DerivedPathOpaque parse(const Store & store, std::string_view);
};
/**
* A derived path that is built from a derivation
*
* Built derived paths are pair of a derivation and some output names.
* They are evaluated by building the derivation, and then replacing the
* output names with the resulting outputs.
*
* Note that does mean a derived store paths evaluates to multiple
* opaque paths, which is sort of icky as expressions are supposed to
* evaluate to single values. Perhaps this should have just a single
* output name.
*/
struct DerivedPathBuilt {
StorePath drvPath;
std::set<std::string> outputs;
std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view);
};
using _DerivedPathRaw = std::variant<
DerivedPathOpaque,
DerivedPathBuilt
>;
/**
* A "derived path" is a very simple sort of expression that evaluates
* to (concrete) store path. It is either:
*
* - opaque, in which case it is just a concrete store path with
* possibly no known derivation
*
* - built, in which case it is a pair of a derivation path and an
* output name.
*/
struct DerivedPath : _DerivedPathRaw {
using Raw = _DerivedPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = DerivedPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
std::string to_string(const Store & store) const;
static DerivedPath parse(const Store & store, std::string_view);
};
/**
* A built derived path with hints in the form of optional concrete output paths.
*
* See 'DerivedPathWithHints' for more an explanation.
*/
struct DerivedPathWithHintsBuilt {
StorePath drvPath;
std::map<std::string, std::optional<StorePath>> outputs;
nlohmann::json toJSON(ref<Store> store) const;
static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view);
};
using _DerivedPathWithHintsRaw = std::variant<
DerivedPath::Opaque,
DerivedPathWithHintsBuilt
>;
/**
* A derived path with hints in the form of optional concrete output paths in the built case.
*
* This type is currently just used by the CLI. The paths are filled in
* during evaluation for derivations that know what paths they will
* produce in advanced, i.e. input-addressed or fixed-output content
* addressed derivations.
*
* That isn't very good, because it puts floating content-addressed
* derivations "at a disadvantage". It would be better to never rely on
* the output path of unbuilt derivations, and exclusively use the
* realizations types to work with built derivations' concrete output
* paths.
*/
// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd.
struct DerivedPathWithHints : _DerivedPathWithHintsRaw {
using Raw = _DerivedPathWithHintsRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = DerivedPathWithHintsBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
};
typedef std::vector<DerivedPathWithHints> DerivedPathsWithHints;
nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store);
}

64
src/libstore/dummy-store.cc Normal file
View file

@ -0,0 +1,64 @@
#include "store-api.hh"
#include "callback.hh"
namespace nix {
struct DummyStoreConfig : virtual StoreConfig {
using StoreConfig::StoreConfig;
const std::string name() override { return "Dummy Store"; }
};
struct DummyStore : public virtual DummyStoreConfig, public virtual Store
{
DummyStore(const std::string scheme, const std::string uri, const Params & params)
: DummyStore(params)
{ }
DummyStore(const Params & params)
: StoreConfig(params)
, DummyStoreConfig(params)
, Store(params)
{ }
string getUri() override
{
return *uriSchemes().begin();
}
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
{
callback(nullptr);
}
static std::set<std::string> uriSchemes() {
return {"dummy"};
}
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override
{ unsupported("addToStore"); }
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); }
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override
{ unsupported("addTextToStore"); }
void narFromPath(const StorePath & path, Sink & sink) override
{ unsupported("narFromPath"); }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
{ unsupported("queryRealisation"); }
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
}

17
src/libstore/export-import.cc
View file

@ -38,14 +38,14 @@ void Store::exportPath(const StorePath & path, Sink & sink)
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
teeSink
<< exportMagic
<< printStorePath(path);
writeStorePaths(*this, teeSink, info->references);
worker_proto::write(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
@ -69,17 +69,18 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
if (magic != exportMagic)
throw Error("Nix archive cannot be imported; wrong format");
ValidPathInfo info(parseStorePath(readString(source)));
auto path = parseStorePath(readString(source));
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
info.references = readStorePaths<StorePathSet>(*this, source);
auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
auto deriver = readString(source);
auto narHash = hashString(htSHA256, *saved.s);
ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *saved.s);
info.references = references;
info.narSize = saved.s->size();
// Ignore optional legacy signature.

48
src/libstore/filetransfer.cc
View file

@ -5,6 +5,7 @@
#include "s3.hh"
#include "compression.hh"
#include "finally.hh"
#include "callback.hh"
#ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>
@ -30,7 +31,7 @@ namespace nix {
FileTransferSettings fileTransferSettings;
static GlobalConfig::Register r1(&fileTransferSettings);
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
std::string resolveUri(const std::string & uri)
{
@ -94,24 +95,27 @@ struct curlFileTransfer : public FileTransfer
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct)
, callback(std::move(callback))
, finalSink([this](const unsigned char * data, size_t len) {
, finalSink([this](std::string_view data) {
if (this->request.dataCallback) {
auto httpStatus = getHTTPStatus();
/* Only write data to the sink if this is a
successful response. */
if (successfulStatuses.count(httpStatus)) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
writtenToSink += data.size();
this->request.dataCallback(data);
}
} else
this->result.data->append((char *) data, len);
this->result.data->append(data);
})
{
if (!request.expectedETag.empty())
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
if (!request.mimeType.empty())
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
for (auto it = request.headers.begin(); it != request.headers.end(); ++it){
requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
}
}
~TransferItem()
@ -144,7 +148,7 @@ struct curlFileTransfer : public FileTransfer
}
LambdaSink finalSink;
std::shared_ptr<CompressionSink> decompressionSink;
std::shared_ptr<FinishSink> decompressionSink;
std::optional<StringSink> errorSink;
std::exception_ptr writeException;
@ -167,8 +171,8 @@ struct curlFileTransfer : public FileTransfer
}
if (errorSink)
(*errorSink)((unsigned char *) contents, realSize);
(*decompressionSink)((unsigned char *) contents, realSize);
(*errorSink)({(char *) contents, realSize});
(*decompressionSink)({(char *) contents, realSize});
return realSize;
} catch (...) {
@ -371,6 +375,13 @@ struct curlFileTransfer : public FileTransfer
else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
{
result.cached = httpStatus == 304;
// In 2021, GitHub responds to If-None-Match with 304,
// but omits ETag. We just use the If-None-Match etag
// since 304 implies they are the same.
if (httpStatus == 304 && result.etag == "")
result.etag = request.expectedETag;
act.progress(result.bodySize, result.bodySize);
done = true;
callback(std::move(result));
@ -628,11 +639,7 @@ struct curlFileTransfer : public FileTransfer
workerThreadMain();
} catch (nix::Interrupted & e) {
} catch (std::exception & e) {
logError({
.name = "File transfer",
.hint = hintfmt("unexpected error in download thread: %s",
e.what())
});
printError("unexpected error in download thread: %s", e.what());
}
{
@ -772,7 +779,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
state->request.notify_one();
});
request.dataCallback = [_state](char * buf, size_t len) {
request.dataCallback = [_state](std::string_view data) {
auto state(_state->lock());
@ -790,7 +797,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
/* Append data to the buffer and wake up the calling
thread. */
state->data.append(buf, len);
state->data.append(data);
state->avail.notify_one();
};
@ -836,7 +843,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
if it's blocked on a full buffer. We don't hold the state
lock while doing this to prevent blocking the download
thread if sink() takes a long time. */
sink((unsigned char *) chunk.data(), chunk.size());
sink(chunk);
}
}
@ -848,11 +855,10 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
} else {
err.hint = hf;
}
if (response && (response->size() < 1024 || response->find("<html>") != string::npos))
err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response));
else
err.msg = hf;
}
bool isUri(const string & s)

32
src/libstore/filetransfer.hh
View file

@ -17,15 +17,30 @@ struct FileTransferSettings : Config
Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix",
"String appended to the user agent in HTTP requests."};
Setting<size_t> httpConnections{this, 25, "http-connections",
"Number of parallel HTTP connections.",
Setting<size_t> httpConnections{
this, 25, "http-connections",
R"(
The maximum number of parallel TCP connections used to fetch
files from binary caches and by other downloads. It defaults
to 25. 0 means no limit.
)",
{"binary-caches-parallel-connections"}};
Setting<unsigned long> connectTimeout{this, 0, "connect-timeout",
"Timeout for connecting to servers during downloads. 0 means use curl's builtin default."};
Setting<unsigned long> connectTimeout{
this, 0, "connect-timeout",
R"(
The timeout (in seconds) for establishing connections in the
binary cache substituter. It corresponds to `curl`’s
`--connect-timeout` option.
)"};
Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout",
"Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."};
Setting<unsigned long> stalledDownloadTimeout{
this, 300, "stalled-download-timeout",
R"(
The timeout (in seconds) for receiving data from servers
during download. Nix cancels idle downloads after this
timeout's duration.
)"};
Setting<unsigned int> tries{this, 5, "download-attempts",
"How often Nix will attempt to download a file before giving up."};
@ -36,6 +51,7 @@ extern FileTransferSettings fileTransferSettings;
struct FileTransferRequest
{
std::string uri;
Headers headers;
std::string expectedETag;
bool verifyTLS = true;
bool head = false;
@ -45,9 +61,9 @@ struct FileTransferRequest
bool decompress = true;
std::shared_ptr<std::string> data;
std::string mimeType;
std::function<void(char *, size_t)> dataCallback;
std::function<void(std::string_view data)> dataCallback;
FileTransferRequest(const std::string & uri)
FileTransferRequest(std::string_view uri)
: uri(uri), parentAct(getCurActivity()) { }
std::string verb()

Some files were not shown because too many files have changed in this diff Show more