nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-28 21:21:00 +01:00
Code Activity

Merge branch 'master' into path-setting

Browse source
This commit is contained in:
John Ericson 2025-11-26 17:57:45 -05:00
commit 37cf990b41
145 changed files with 2949 additions and 625 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/json-schema-checks/derivation-options Symbolic link
View file

@ -0,0 +1 @@
../libstore-tests/data/derivation

25
src/json-schema-checks/meson.build
View file

@ -71,6 +71,18 @@ schemas = [
'with-signature.json',
],
},
{
'stem' : 'derivation-options',
'schema' : schema_dir / 'derivation-options-v1.yaml',
'files' : [
'ia' / 'defaults.json',
'ia' / 'all_set.json',
'ia' / 'structuredAttrs_defaults.json',
'ia' / 'structuredAttrs_all_set.json',
'ca' / 'all_set.json',
'ca' / 'structuredAttrs_all_set.json',
],
},
]
# Derivation and Derivation output
@ -200,6 +212,19 @@ schemas += [
},
]
# Dummy store
schemas += [
{
'stem' : 'store',
'schema' : schema_dir / 'store-v1.yaml',
'files' : [
'empty.json',
'one-flat-file.json',
'one-derivation.json',
],
},
]
# Validate each example against the schema
foreach schema : schemas
stem = schema['stem']

1
src/json-schema-checks/package.nix
View file

@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: {
../../src/libstore-tests/data/path-info
../../src/libstore-tests/data/nar-info
../../src/libstore-tests/data/build-result
../../src/libstore-tests/data/dummy-store
./.
];

1
src/json-schema-checks/store Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/dummy-store

6
src/libcmd/command.cc
View file

@ -297,7 +297,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
MixDefaultProfile::MixDefaultProfile()
{
profile = getDefaultProfile();
profile = getDefaultProfile().string();
}
MixEnvironment::MixEnvironment()
@ -391,7 +391,7 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu
auto symlink = outLink;
if (i)
symlink += fmt("-%d", i);
store.addPermRoot(bo.path, absPath(symlink.string()));
store.addPermRoot(bo.path, absPath(symlink).string());
},
[&](const BuiltPath::Built & bfd) {
for (auto & output : bfd.outputs) {
@ -400,7 +400,7 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu
symlink += fmt("-%d", i);
if (output.first != "out")
symlink += fmt("-%s", output.first);
store.addPermRoot(output.second, absPath(symlink.string()));
store.addPermRoot(output.second, absPath(symlink).string());
}
},
},

9
src/libcmd/common-eval-args.cc
View file

@ -165,7 +165,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
state.parseExprFromString(
arg.expr,
compatibilitySettings.nixShellShebangArgumentsRelativeToScript
? state.rootPath(absPath(getCommandBaseDir()))
? state.rootPath(absPath(getCommandBaseDir()).string())
: state.rootPath(".")));
},
[&](const AutoArgString & arg) { v->mkString(arg.s, state.mem); },
@ -177,7 +177,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
return res.finish();
}
SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
SourcePath lookupFileArg(EvalState & state, std::string_view s, const std::filesystem::path * baseDir)
{
if (EvalSettings::isPseudoUrl(s)) {
auto accessor = fetchers::downloadTarball(*state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
@ -197,12 +197,13 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
}
else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p(s.substr(1, s.size() - 2));
// Should perhaps be a `CanonPath`?
std::string p(s.substr(1, s.size() - 2));
return state.findFile(p);
}
else
return state.rootPath(baseDir ? absPath(s, *baseDir) : absPath(s));
return state.rootPath(absPath(std::filesystem::path{s}, baseDir).string());
}
} // namespace nix

4
src/libcmd/include/nix/cmd/command.hh
View file

@ -134,7 +134,7 @@ struct MixFlakeOptions : virtual Args, EvalCommand
struct SourceExprCommand : virtual Args, MixFlakeOptions
{
std::optional<Path> file;
std::optional<std::filesystem::path> file;
std::optional<std::string> expr;
SourceExprCommand();
@ -310,7 +310,7 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
struct MixProfile : virtual StoreCommand
{
std::optional<Path> profile;
std::optional<std::filesystem::path> profile;
MixProfile();

2
src/libcmd/include/nix/cmd/common-eval-args.hh
View file

@ -84,6 +84,6 @@ private:
/**
* @param baseDir Optional [base directory](https://nix.dev/manual/nix/development/glossary#gloss-base-directory)
*/
SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);
SourcePath lookupFileArg(EvalState & state, std::string_view s, const std::filesystem::path * baseDir = nullptr);
} // namespace nix

2
src/libcmd/include/nix/cmd/installable-value.hh
View file

@ -17,7 +17,7 @@ class AttrCursor;
struct App
{
std::vector<DerivedPath> context;
Path program;
std::filesystem::path program;
// FIXME: add args, sandbox settings, metadata, ...
};

11
src/libcmd/include/nix/cmd/repl.hh
View file

@ -19,7 +19,16 @@ struct AbstractNixRepl
typedef std::vector<std::pair<Value *, std::string>> AnnotatedValues;
using RunNix = void(Path program, const Strings & args, const std::optional<std::string> & input);
/**
* Run a nix executable
*
* @todo this is a layer violation
*
* @param programName Name of the command, e.g. `nix` or `nix-env`.
* @param args aguments to the command.
*/
using RunNix =
void(const std::string & programName, const Strings & args, const std::optional<std::string> & input);
/**
* @param runNix Function to run the nix CLI to support various

18
src/libcmd/installables.cc
View file

@ -132,7 +132,7 @@ MixFlakeOptions::MixFlakeOptions()
lockFlags.writeLockFile = false;
lockFlags.inputOverrides.insert_or_assign(
flake::parseInputAttrPath(inputAttrPath),
parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true));
parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true));
}},
.completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
if (n == 0) {
@ -173,7 +173,7 @@ MixFlakeOptions::MixFlakeOptions()
auto flake = flake::lockFlake(
flakeSettings,
*evalState,
parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())),
parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string()),
{.writeLockFile = false});
for (auto & [inputName, input] : flake.lockFile.root->inputs) {
auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
@ -263,7 +263,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
evalSettings.pureEval = false;
auto state = getEvalState();
auto e = state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, *file)));
auto e = state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, file->string())));
Value root;
state->eval(e, root);
@ -465,10 +465,10 @@ Installables SourceExprCommand::parseInstallables(ref<Store> store, std::vector<
state->eval(e, *vFile);
} else if (file) {
auto dir = absPath(getCommandBaseDir());
state->evalFile(lookupFileArg(*state, *file, &dir), *vFile);
state->evalFile(lookupFileArg(*state, file->string(), &dir), *vFile);
} else {
Path dir = absPath(getCommandBaseDir());
auto e = state->parseExprFromString(*expr, state->rootPath(dir));
auto dir = absPath(getCommandBaseDir());
auto e = state->parseExprFromString(*expr, state->rootPath(dir.string()));
state->eval(e, *vFile);
}
@ -801,7 +801,8 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
std::vector<FlakeRef> res;
res.reserve(rawInstallables.size());
for (const auto & i : rawInstallables)
res.push_back(parseFlakeRefWithFragment(fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first);
res.push_back(
parseFlakeRefWithFragment(fetchSettings, expandTilde(i), absPath(getCommandBaseDir()).string()).first);
return res;
}
@ -820,7 +821,8 @@ void RawInstallablesCommand::run(ref<Store> store)
std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
{
return {parseFlakeRefWithFragment(fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first};
return {parseFlakeRefWithFragment(fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir()).string())
.first};
}
void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables)

19
src/libcmd/repl.cc
View file

@ -58,8 +58,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
{
size_t debugTraceIndex;
// Arguments passed to :load, saved so they can be reloaded with :reload
Strings loadedFiles;
std::list<std::filesystem::path> loadedFiles;
// Arguments passed to :load-flake, saved so they can be reloaded with :reload
Strings loadedFlakes;
std::function<AnnotatedValues()> getValues;
@ -73,7 +72,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
RunNix * runNixPtr;
void runNix(Path program, const Strings & args, const std::optional<std::string> & input = {});
void runNix(const std::string & program, const Strings & args, const std::optional<std::string> & input = {});
std::unique_ptr<ReplInteracter> interacter;
@ -92,7 +91,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
StorePath getDerivationPath(Value & v);
ProcessLineResult processLine(std::string line);
void loadFile(const Path & path);
void loadFile(const std::filesystem::path & path);
void loadFlake(const std::string & flakeRef);
void loadFiles();
void loadFlakes();
@ -539,7 +538,9 @@ ProcessLineResult NixRepl::processLine(std::string line)
Value v;
evalString(arg, v);
StorePath drvPath = getDerivationPath(v);
Path drvPathRaw = state->store->printStorePath(drvPath);
// N.B. This need not be a local / native file path. For
// example, we might be using an SSH store to a different OS.
std::string drvPathRaw = state->store->printStorePath(drvPath);
if (command == ":b" || command == ":bl") {
state->store->buildPaths({
@ -712,12 +713,12 @@ ProcessLineResult NixRepl::processLine(std::string line)
return ProcessLineResult::PromptAgain;
}
void NixRepl::loadFile(const Path & path)
void NixRepl::loadFile(const std::filesystem::path & path)
{
loadedFiles.remove(path);
loadedFiles.push_back(path);
Value v, v2;
state->evalFile(lookupFileArg(*state, path), v);
state->evalFile(lookupFileArg(*state, path.string()), v);
state->autoCallFunction(*autoArgs, v, v2);
addAttrsToScope(v2);
}
@ -790,7 +791,7 @@ void NixRepl::reloadFilesAndFlakes()
void NixRepl::loadFiles()
{
Strings old = loadedFiles;
decltype(loadedFiles) old = loadedFiles;
loadedFiles.clear();
for (auto & i : old) {
@ -888,7 +889,7 @@ void NixRepl::evalString(std::string s, Value & v)
state->forceValue(v, v.determinePos(noPos));
}
void NixRepl::runNix(Path program, const Strings & args, const std::optional<std::string> & input)
void NixRepl::runNix(const std::string & program, const Strings & args, const std::optional<std::string> & input)
{
if (runNixPtr)
(*runNixPtr)(program, args, input);

8
src/libexpr-tests/nix_api_expr.cc
View file

@ -18,13 +18,13 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path)
{
auto tmpDir = nix::createTempDir();
auto delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
auto nixpkgs = tmpDir + "/pkgs";
auto nixos = tmpDir + "/cfg";
auto nixpkgs = tmpDir / "pkgs";
auto nixos = tmpDir / "cfg";
std::filesystem::create_directories(nixpkgs);
std::filesystem::create_directories(nixos);
std::string nixpkgsEntry = "nixpkgs=" + nixpkgs;
std::string nixosEntry = "nixos-config=" + nixos;
std::string nixpkgsEntry = "nixpkgs=" + nixpkgs.string();
std::string nixosEntry = "nixos-config=" + nixos.string();
const char * lookupPath[] = {nixpkgsEntry.c_str(), nixosEntry.c_str(), nullptr};
auto builder = nix_eval_state_builder_new(ctx, store);

16
src/libexpr/eval-settings.cc
View file

@ -60,18 +60,18 @@ EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lo
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
auto add = [&](const Path & p, const std::string & s = std::string()) {
auto add = [&](const std::filesystem::path & p, const std::string & s = std::string()) {
if (std::filesystem::exists(p)) {
if (s.empty()) {
res.push_back(p);
res.push_back(p.string());
} else {
res.push_back(s + "=" + p);
res.push_back(s + "=" + p.string());
}
}
};
add(getNixDefExpr() + "/channels");
add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
add(std::filesystem::path{getNixDefExpr()} / "channels");
add(rootChannelsDir() / "nixpkgs", "nixpkgs");
add(rootChannelsDir());
return res;
@ -103,9 +103,9 @@ const std::string & EvalSettings::getCurrentSystem() const
return evalSystem != "" ? evalSystem : settings.thisSystem.get();
}
Path getNixDefExpr()
std::filesystem::path getNixDefExpr()
{
return settings.useXDGBaseDirectories ? getStateDir() + "/defexpr" : getHome() + "/.nix-defexpr";
return settings.useXDGBaseDirectories ? getStateDir() / "defexpr" : getHome() / ".nix-defexpr";
}
} // namespace nix
} // namespace nix

2
src/libexpr/include/nix/expr/eval-settings.hh
View file

@ -366,6 +366,6 @@ struct EvalSettings : Config
/**
* Conventionally part of the default nix path in impure mode.
*/
Path getNixDefExpr();
std::filesystem::path getNixDefExpr();
} // namespace nix

2
src/libexpr/include/nix/expr/nixexpr.hh
View file

@ -438,6 +438,7 @@ struct ExprAttrs : Expr
std::shared_ptr<const StaticEnv> bindInheritSources(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
Env * buildInheritFromEnv(EvalState & state, Env & up);
void showBindings(const SymbolTable & symbols, std::ostream & str) const;
void moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc);
};
struct ExprList : Expr
@ -622,6 +623,7 @@ struct ExprCall : Expr
virtual void resetCursedOr() override;
virtual void warnIfCursedOr(const SymbolTable & symbols, const PosTable & positions) override;
void moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc);
COMMON_METHODS
};

73
src/libexpr/include/nix/expr/parser-state.hh
View file

@ -47,6 +47,79 @@ struct ParserLocation
}
};
/**
* This represents a string-like parse that possibly has yet to be constructed.
*
* Examples:
* "foo"
* ${"foo" + "bar"}
* "foo.bar"
* "foo-${a}"
*
* Using this type allows us to avoid construction altogether in cases where what we actually need is the string
* contents. For example in foo."bar.baz", there is no need to construct an AST node for "bar.baz", but we don't know
* that until we bubble the value up during parsing and see that it's a node in an AttrPath.
*/
class ToBeStringyExpr
{
private:
using Raw = std::variant<std::monostate, std::string_view, Expr *>;
Raw raw;
public:
ToBeStringyExpr() = default;
ToBeStringyExpr(std::string_view v)
: raw(v)
{
}
ToBeStringyExpr(Expr * expr)
: raw(expr)
{
assert(expr);
}
/**
* Visits the expression and invokes an overloaded functor object \ref f.
* If the underlying Expr has a dynamic type of ExprString the overload taking std::string_view
* is invoked.
*
* Used to consistently handle simple StringExpr ${"string"} as non-dynamic attributes.
* @see https://github.com/NixOS/nix/issues/14642
*/
template<class F>
void visit(F && f)
{
std::visit(
overloaded{
[&](std::string_view str) { f(str); },
[&](Expr * expr) {
ExprString * str = dynamic_cast<ExprString *>(expr);
if (str)
f(str->v.string_view());
else
f(expr);
},
[](std::monostate) { unreachable(); }},
raw);
}
/**
* Get or create an Expr from either an existing Expr or from a string.
* Delays the allocation or an AST node in case the parser only cares about string contents.
*/
Expr * toExpr(Exprs & exprs)
{
return std::visit(
overloaded{
[&](std::string_view str) -> Expr * { return exprs.add<ExprString>(exprs.alloc, str); },
[&](Expr * expr) { return expr; },
[](std::monostate) -> Expr * { unreachable(); }},
raw);
}
};
struct LexerState
{
/**

35
src/libexpr/nixexpr.cc
View file

@ -399,18 +399,19 @@ ExprAttrs::bindInheritSources(EvalState & es, const std::shared_ptr<const Static
return inner;
}
void ExprAttrs::moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc)
{
AttrDefs newAttrs{std::move(*attrs), alloc};
attrs.emplace(std::move(newAttrs), alloc);
DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), alloc};
dynamicAttrs.emplace(std::move(newDynamicAttrs), alloc);
if (inheritFromExprs)
inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), alloc);
}
void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
// Move storage into the Exprs arena
{
auto arena = es.mem.exprs.alloc;
AttrDefs newAttrs{std::move(*attrs), arena};
attrs.emplace(std::move(newAttrs), arena);
DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), arena};
dynamicAttrs.emplace(std::move(newDynamicAttrs), arena);
if (inheritFromExprs)
inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), arena);
}
moveDataToAllocator(es.mem.exprs.alloc);
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
@ -484,14 +485,15 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
body->bindVars(es, newEnv);
}
void ExprCall::moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc)
{
std::pmr::vector<Expr *> newArgs{std::move(*args), alloc};
args.emplace(std::move(newArgs), alloc);
}
void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
// Move storage into the Exprs arena
{
auto arena = es.mem.exprs.alloc;
std::pmr::vector<Expr *> newArgs{std::move(*args), arena};
args.emplace(std::move(newArgs), arena);
}
moveDataToAllocator(es.mem.exprs.alloc);
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
@ -502,6 +504,7 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{
attrs->moveDataToAllocator(es.mem.exprs.alloc);
auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs->size());

45
src/libexpr/parser.y
View file

@ -138,7 +138,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
%type <std::vector<std::pair<PosIdx, Expr *>>> string_parts_interpolated
%type <std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>> ind_string_parts
%type <Expr *> path_start
%type <std::variant<Expr *, std::string_view>> string_parts string_attr
%type <ToBeStringyExpr> string_parts string_attr
%type <StringToken> attr
%token <StringToken> ID
%token <StringToken> STR IND_STR
@ -297,12 +297,7 @@ expr_simple
}
| INT_LIT { $$ = state->exprs.add<ExprInt>($1); }
| FLOAT_LIT { $$ = state->exprs.add<ExprFloat>($1); }
| '"' string_parts '"' {
std::visit(overloaded{
[&](std::string_view str) { $$ = state->exprs.add<ExprString>(state->exprs.alloc, str); },
[&](Expr * expr) { $$ = expr; }},
$2);
}
| '"' string_parts '"' { $$ = $2.toExpr(state->exprs); }
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
$$ = state->stripIndentation(CUR_POS, $2);
}
@ -342,9 +337,9 @@ expr_simple
;
string_parts
: STR { $$ = $1; }
| string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1); }
| { $$ = std::string_view(); }
: STR { $$ = {$1}; }
| string_parts_interpolated { $$ = {state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1)}; }
| { $$ = {std::string_view()}; }
;
string_parts_interpolated
@ -389,7 +384,7 @@ path_start
std::string_view($1.p, $1.l)
);
}
Path path(getHome() + std::string($1.p + 1, $1.l - 1));
Path path(getHome().string() + std::string($1.p + 1, $1.l - 1));
$$ = state->exprs.add<ExprPath>(state->exprs.alloc, ref<SourceAccessor>(state->rootFS), path);
}
;
@ -447,15 +442,15 @@ attrs
: attrs attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($2), state->at(@2)); }
| attrs string_attr
{ $$ = std::move($1);
std::visit(overloaded {
$2.visit(overloaded{
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str), state->at(@2)); },
[&](Expr * expr) {
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}
}, $2);
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}}
);
}
| { }
;
@ -464,17 +459,17 @@ attrpath
: attrpath '.' attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($3)); }
| attrpath '.' string_attr
{ $$ = std::move($1);
std::visit(overloaded {
$3.visit(overloaded{
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($3));
[&](Expr * expr) { $$.emplace_back(expr); }}
);
}
| attr { $$.emplace_back(state->symbols.create($1)); }
| string_attr
{ std::visit(overloaded {
{ $1.visit(overloaded{
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($1));
[&](Expr * expr) { $$.emplace_back(expr); }}
);
}
;
@ -485,7 +480,7 @@ attr
string_attr
: '"' string_parts '"' { $$ = std::move($2); }
| DOLLAR_CURLY expr '}' { $$ = $2; }
| DOLLAR_CURLY expr '}' { $$ = {$2}; }
;
list

23
src/libexpr/primops.cc
View file

@ -1774,28 +1774,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{});
}
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
switch (hashModulo.kind) {
case DrvHash::Kind::Regular:
for (auto & i : outputs) {
auto h = get(hashModulo.hashes, i);
if (!h)
state.error<AssertionError>("derivation produced no hash for output '%s'", i).atPos(v).debugThrow();
auto outPath = state.store->makeOutputPath(i, *h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
i,
DerivationOutput::InputAddressed{
.path = std::move(outPath),
});
}
break;
;
case DrvHash::Kind::Deferred:
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{});
}
}
drv.fillInOutputPaths(*state.store);
}
/* Write the resulting term into the Nix store directory. */

8
src/libfetchers/git-lfs-fetch.cc
View file

@ -268,10 +268,10 @@ void Fetch::fetch(
return;
}
Path cacheDir = getCacheDir() + "/git-lfs";
std::filesystem::path cacheDir = getCacheDir() / "git-lfs";
std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false)
+ "/" + pointer->oid;
Path cachePath = cacheDir + "/" + key;
std::filesystem::path cachePath = cacheDir / key;
if (pathExists(cachePath)) {
debug("using cache entry %s -> %s", key, cachePath);
sink(readFile(cachePath));
@ -302,8 +302,8 @@ void Fetch::fetch(
downloadToSink(ourl, authHeader, sink, sha256, size);
debug("creating cache entry %s -> %s", key, cachePath);
if (!pathExists(dirOf(cachePath)))
createDirs(dirOf(cachePath));
if (!pathExists(cachePath.parent_path()))
createDirs(cachePath.parent_path());
writeFile(cachePath, sink.s);
debug("%s fetched with git-lfs", pointerFilePath);

2
src/libfetchers/git-utils.cc
View file

@ -206,7 +206,7 @@ static void initRepoAtomically(std::filesystem::path & path, bool bare)
if (pathExists(path.string()))
return;
Path tmpDir = createTempDir(os_string_to_string(PathViewNG{std::filesystem::path(path).parent_path()}));
std::filesystem::path tmpDir = createTempDir(path.parent_path());
AutoDelete delTmpDir(tmpDir, true);
Repository tmpRepo;

14
src/libfetchers/git.cc
View file

@ -42,10 +42,10 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st)
return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
}
Path getCachePath(std::string_view key, bool shallow)
std::filesystem::path getCachePath(std::string_view key, bool shallow)
{
return getCacheDir() + "/gitv3/" + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false)
+ (shallow ? "-shallow" : "");
return getCacheDir() / "gitv3"
/ (hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : ""));
}
// Returns the name of the HEAD branch.
@ -55,7 +55,7 @@ Path getCachePath(std::string_view key, bool shallow)
//
// ref: refs/heads/main HEAD
// ...
std::optional<std::string> readHead(const Path & path)
std::optional<std::string> readHead(const std::filesystem::path & path)
{
auto [status, output] = runProgram(
RunOptions{
@ -86,7 +86,7 @@ std::optional<std::string> readHead(const Path & path)
// Persist the HEAD ref from the remote repo in the local cached repo.
bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::string & headRef)
{
Path cacheDir = getCachePath(actualUrl, shallow);
std::filesystem::path cacheDir = getCachePath(actualUrl, shallow);
try {
runProgram("git", true, {"-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef});
} catch (ExecError & e) {
@ -109,8 +109,8 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl, bool sh
{
// Create a cache path to store the branch of the HEAD ref. Append something
// in front of the URL to prevent collision with the repository itself.
Path cacheDir = getCachePath(actualUrl, shallow);
Path headRefFile = cacheDir + "/HEAD";
std::filesystem::path cacheDir = getCachePath(actualUrl, shallow);
std::filesystem::path headRefFile = cacheDir / "HEAD";
time_t now = time(0);
struct stat st;

63
src/libfetchers/github.cc
View file

@ -41,18 +41,16 @@ struct GitArchiveInputScheme : InputScheme
/* This ignores empty path segments for back-compat. Older versions used a tokenizeString here. */
auto path = url.pathSegments(/*skipEmpty=*/true) | std::ranges::to<std::vector<std::string>>();
std::optional<Hash> rev;
std::optional<std::string> rev;
std::optional<std::string> ref;
std::optional<std::string> host_url;
auto size = path.size();
if (size == 3) {
if (std::regex_match(path[2], revRegex))
rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
else if (isLegalRefName(path[2]))
ref = path[2];
rev = path[2];
else
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url, path[2]);
ref = path[2];
} else if (size > 3) {
std::string rs;
for (auto i = std::next(path.begin(), 2); i != path.end(); i++) {
@ -61,12 +59,7 @@ struct GitArchiveInputScheme : InputScheme
rs += "/";
}
}
if (isLegalRefName(rs)) {
ref = rs;
} else {
throw BadURL("in URL '%s', '%s' is not a branch/tag name", url, rs);
}
ref = rs;
} else if (size < 2)
throw BadURL("URL '%s' is invalid", url);
@ -74,40 +67,32 @@ struct GitArchiveInputScheme : InputScheme
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url);
rev = Hash::parseAny(value, HashAlgorithm::SHA1);
rev = value;
} else if (name == "ref") {
if (!isLegalRefName(value))
throw BadURL("URL '%s' contains an invalid branch/tag name", url);
if (ref)
throw BadURL("URL '%s' contains multiple branch/tag names", url);
ref = value;
} else if (name == "host") {
if (!std::regex_match(value, hostRegex))
throw BadURL("URL '%s' contains an invalid instance host", url);
} else if (name == "host")
host_url = value;
}
// FIXME: barf on unsupported attributes
}
if (ref && rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev());
Input input{};
input.attrs.insert_or_assign("type", std::string{schemeName()});
input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]);
Attrs attrs;
attrs.insert_or_assign("type", std::string{schemeName()});
attrs.insert_or_assign("owner", path[0]);
attrs.insert_or_assign("repo", path[1]);
if (rev)
input.attrs.insert_or_assign("rev", rev->gitRev());
attrs.insert_or_assign("rev", *rev);
if (ref)
input.attrs.insert_or_assign("ref", *ref);
attrs.insert_or_assign("ref", *ref);
if (host_url)
input.attrs.insert_or_assign("host", *host_url);
attrs.insert_or_assign("host", *host_url);
auto narHash = url.query.find("narHash");
if (narHash != url.query.end())
input.attrs.insert_or_assign("narHash", narHash->second);
attrs.insert_or_assign("narHash", narHash->second);
return input;
return inputFromAttrs(settings, attrs);
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
@ -154,6 +139,24 @@ struct GitArchiveInputScheme : InputScheme
getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo");
auto ref = maybeGetStrAttr(attrs, "ref");
auto rev = maybeGetStrAttr(attrs, "rev");
if (ref && rev)
throw BadURL(
"input %s contains both a commit hash ('%s') and a branch/tag name ('%s')",
attrsToJSON(attrs),
*rev,
*ref);
if (rev)
Hash::parseAny(*rev, HashAlgorithm::SHA1);
if (ref && !isLegalRefName(*ref))
throw BadURL("input %s contains an invalid branch/tag name", attrsToJSON(attrs));
if (auto host = maybeGetStrAttr(attrs, "host"); host && !std::regex_match(*host, hostRegex))
throw BadURL("input %s contains an invalid instance host", attrsToJSON(attrs));
Input input{};
input.attrs = attrs;
return input;

6
src/libfetchers/include/nix/fetchers/registry.hh
View file

@ -39,7 +39,7 @@ struct Registry
static std::shared_ptr<Registry> read(const Settings & settings, const SourcePath & path, RegistryType type);
void write(const Path & path);
void write(const std::filesystem::path & path);
void add(const Input & from, const Input & to, const Attrs & extraAttrs);
@ -50,9 +50,9 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
std::shared_ptr<Registry> getUserRegistry(const Settings & settings);
std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Path & p);
std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const std::filesystem::path & p);
Path getUserRegistryPath();
std::filesystem::path getUserRegistryPath();
Registries getRegistries(const Settings & settings, Store & store);

24
src/libfetchers/mercurial.cc
View file

@ -213,11 +213,11 @@ struct MercurialInputScheme : InputScheme
runHg({"status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0"}),
"\0"s);
Path actualPath(absPath(actualUrl));
std::filesystem::path actualPath(absPath(actualUrl));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualPath));
std::string file(p, actualPath.size() + 1);
assert(hasPrefix(p, actualPath.string()));
std::string file(p, actualPath.string().size() + 1);
auto st = lstat(p);
@ -232,7 +232,7 @@ struct MercurialInputScheme : InputScheme
auto storePath = store.addToStore(
input.getName(),
{getFSSourceAccessor(), CanonPath(actualPath)},
{getFSSourceAccessor(), CanonPath(actualPath.string())},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256,
{},
@ -275,10 +275,8 @@ struct MercurialInputScheme : InputScheme
return makeResult(res->value, res->storePath);
}
Path cacheDir =
fmt("%s/hg/%s",
getCacheDir(),
hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false));
std::filesystem::path cacheDir =
getCacheDir() / "hg" / hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false);
/* If this is a commit hash that we already have, we don't
have to pull again. */
@ -292,7 +290,7 @@ struct MercurialInputScheme : InputScheme
try {
runHg({"pull", "-R", cacheDir, "--", actualUrl});
} catch (ExecError & e) {
auto transJournal = cacheDir + "/.hg/store/journal";
auto transJournal = cacheDir / ".hg" / "store" / "journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runHg({"recover", "-R", cacheDir});
@ -302,7 +300,7 @@ struct MercurialInputScheme : InputScheme
}
}
} else {
createDirs(dirOf(cacheDir));
createDirs(dirOf(cacheDir.string()));
runHg({"clone", "--noupdate", "--", actualUrl, cacheDir});
}
}
@ -328,14 +326,14 @@ struct MercurialInputScheme : InputScheme
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), store))
return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir();
std::filesystem::path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runHg({"archive", "-R", cacheDir, "-r", rev.gitRev(), tmpDir});
deletePath(tmpDir + "/.hg_archival.txt");
deletePath(tmpDir / ".hg_archival.txt");
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())});
Attrs infoAttrs({
{"revCount", (uint64_t) revCount},

22
src/libfetchers/registry.cc
View file

@ -56,7 +56,7 @@ std::shared_ptr<Registry> Registry::read(const Settings & settings, const Source
return registry;
}
void Registry::write(const Path & path)
void Registry::write(const std::filesystem::path & path)
{
nlohmann::json arr;
for (auto & entry : entries) {
@ -74,7 +74,7 @@ void Registry::write(const Path & path)
json["version"] = 2;
json["flakes"] = std::move(arr);
createDirs(dirOf(path));
createDirs(path.parent_path());
writeFile(path, json.dump(2));
}
@ -90,38 +90,38 @@ void Registry::remove(const Input & input)
entries.end());
}
static Path getSystemRegistryPath()
static std::filesystem::path getSystemRegistryPath()
{
return settings.nixConfDir + "/registry.json";
return settings.nixConfDir / "registry.json";
}
static std::shared_ptr<Registry> getSystemRegistry(const Settings & settings)
{
static auto systemRegistry = Registry::read(
settings,
SourcePath{getFSSourceAccessor(), CanonPath{getSystemRegistryPath()}}.resolveSymlinks(),
SourcePath{getFSSourceAccessor(), CanonPath{getSystemRegistryPath().string()}}.resolveSymlinks(),
Registry::System);
return systemRegistry;
}
Path getUserRegistryPath()
std::filesystem::path getUserRegistryPath()
{
return getConfigDir() + "/registry.json";
return getConfigDir() / "registry.json";
}
std::shared_ptr<Registry> getUserRegistry(const Settings & settings)
{
static auto userRegistry = Registry::read(
settings,
SourcePath{getFSSourceAccessor(), CanonPath{getUserRegistryPath()}}.resolveSymlinks(),
SourcePath{getFSSourceAccessor(), CanonPath{getUserRegistryPath().string()}}.resolveSymlinks(),
Registry::User);
return userRegistry;
}
std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Path & p)
std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const std::filesystem::path & p)
{
static auto customRegistry =
Registry::read(settings, SourcePath{getFSSourceAccessor(), CanonPath{p}}.resolveSymlinks(), Registry::Custom);
static auto customRegistry = Registry::read(
settings, SourcePath{getFSSourceAccessor(), CanonPath{p.string()}}.resolveSymlinks(), Registry::Custom);
return customRegistry;
}

22
src/libflake-tests/nix_api_flake.cc
View file

@ -86,7 +86,7 @@ TEST_F(nix_api_store_test, nix_api_load_flake)
auto tmpDir = nix::createTempDir();
nix::AutoDelete delTmpDir(tmpDir, true);
nix::writeFile(tmpDir + "/flake.nix", R"(
nix::writeFile(tmpDir / "flake.nix", R"(
{
outputs = { ... }: {
hello = "potato";
@ -121,7 +121,8 @@ TEST_F(nix_api_store_test, nix_api_load_flake)
assert_ctx_ok();
ASSERT_NE(nullptr, parseFlags);
auto r0 = nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.size());
auto r0 =
nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.string().size());
assert_ctx_ok();
ASSERT_EQ(NIX_OK, r0);
@ -177,8 +178,8 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags)
auto tmpDir = nix::createTempDir();
nix::AutoDelete delTmpDir(tmpDir, true);
nix::createDirs(tmpDir + "/b");
nix::writeFile(tmpDir + "/b/flake.nix", R"(
nix::createDirs(tmpDir / "b");
nix::writeFile(tmpDir / "b" / "flake.nix", R"(
{
outputs = { ... }: {
hello = "BOB";
@ -186,18 +187,18 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags)
}
)");
nix::createDirs(tmpDir + "/a");
nix::writeFile(tmpDir + "/a/flake.nix", R"(
nix::createDirs(tmpDir / "a");
nix::writeFile(tmpDir / "a" / "flake.nix", R"(
{
inputs.b.url = ")" + tmpDir + R"(/b";
inputs.b.url = ")" + tmpDir.string() + R"(/b";
outputs = { b, ... }: {
hello = b.hello;
};
}
)");
nix::createDirs(tmpDir + "/c");
nix::writeFile(tmpDir + "/c/flake.nix", R"(
nix::createDirs(tmpDir / "c");
nix::writeFile(tmpDir / "c" / "flake.nix", R"(
{
outputs = { ... }: {
hello = "Claire";
@ -230,7 +231,8 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags)
assert_ctx_ok();
ASSERT_NE(nullptr, parseFlags);
auto r0 = nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.size());
auto r0 =
nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.string().size());
assert_ctx_ok();
ASSERT_EQ(NIX_OK, r0);

22
src/libmain/progress-bar.cc
View file

@ -470,7 +470,8 @@ public:
std::string res;
auto renderActivity =
[&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
[&] [[nodiscard]] (
ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
auto & act = state.activitiesByType[type];
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
for (auto & j : act.its) {
@ -514,7 +515,7 @@ public:
return s;
};
auto renderSizeActivity = [&](ActivityType type, const std::string & itemFmt = "%s") {
auto renderSizeActivity = [&] [[nodiscard]] (ActivityType type, const std::string & itemFmt = "%s") {
auto & act = state.activitiesByType[type];
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
for (auto & j : act.its) {
@ -573,14 +574,17 @@ public:
return s;
};
auto maybeAppendToResult = [&](std::string_view s) {
if (s.empty())
return;
if (!res.empty())
res += ", ";
res += s;
};
auto showActivity =
[&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
auto s = renderActivity(type, itemFmt, numberFmt, unit);
if (s.empty())
return;
if (!res.empty())
res += ", ";
res += s;
maybeAppendToResult(renderActivity(type, itemFmt, numberFmt, unit));
};
showActivity(actBuilds, "%s built");
@ -602,7 +606,7 @@ public:
}
}
renderSizeActivity(actFileTransfer, "%s DL");
maybeAppendToResult(renderSizeActivity(actFileTransfer, "%s DL"));
{
auto s = renderActivity(actOptimiseStore, "%s paths optimised");

96
src/libstore-c/nix_api_store.cc
View file

@ -1,3 +1,6 @@
#include <cstring>
#include <span>
#include "nix_api_store.h"
#include "nix_api_store_internal.h"
#include "nix_api_util.h"
@ -8,6 +11,7 @@
#include "nix/store/store-open.hh"
#include "nix/store/build-result.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/util/base-nix-32.hh"
#include "nix/store/globals.hh"
@ -215,7 +219,65 @@ void nix_derivation_free(nix_derivation * drv)
StorePath * nix_store_path_clone(const StorePath * p)
{
return new StorePath{p->path};
try {
return new StorePath{p->path};
} catch (...) {
return nullptr;
}
}
} // extern "C"
template<size_t S>
static auto to_cpp_array(const uint8_t (&r)[S])
{
return reinterpret_cast<const std::array<std::byte, S> &>(r);
}
extern "C" {
nix_err
nix_store_path_hash(nix_c_context * context, const StorePath * store_path, nix_store_path_hash_part * hash_part_out)
{
try {
auto hashPart = store_path->path.hashPart();
// Decode from Nix32 (base32) encoding to raw bytes
auto decoded = nix::BaseNix32::decode(hashPart);
assert(decoded.size() == sizeof(hash_part_out->bytes));
std::memcpy(hash_part_out->bytes, decoded.data(), sizeof(hash_part_out->bytes));
return NIX_OK;
}
NIXC_CATCH_ERRS
}
StorePath * nix_store_create_from_parts(
nix_c_context * context, const nix_store_path_hash_part * hash, const char * name, size_t name_len)
{
if (context)
context->last_err_code = NIX_OK;
try {
// Encode the 20 raw bytes to Nix32 (base32) format
auto hashStr = nix::BaseNix32::encode(std::span<const std::byte>{to_cpp_array(hash->bytes)});
// Construct the store path basename: <hash>-<name>
std::string baseName;
baseName += hashStr;
baseName += "-";
baseName += std::string_view{name, name_len};
return new StorePath{nix::StorePath(std::move(baseName))};
}
NIXC_CATCH_ERRS_NULL
}
nix_derivation * nix_derivation_clone(const nix_derivation * d)
{
try {
return new nix_derivation{d->drv};
} catch (...) {
return nullptr;
}
}
nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json)
@ -223,17 +285,25 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
if (context)
context->last_err_code = NIX_OK;
try {
auto drv = static_cast<nix::Derivation>(nlohmann::json::parse(json));
auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true);
drv.checkInvariants(*store->ptr, drvPath);
return new nix_derivation{drv};
return new nix_derivation{nix::Derivation::parseJsonAndValidate(*store->ptr, nlohmann::json::parse(json))};
}
NIXC_CATCH_ERRS_NULL
}
nix_err nix_derivation_to_json(
nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata)
{
if (context)
context->last_err_code = NIX_OK;
try {
auto result = static_cast<nlohmann::json>(drv->drv).dump();
if (callback) {
callback(result.data(), result.size(), userdata);
}
}
NIXC_CATCH_ERRS
}
StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation)
{
if (context)
@ -258,4 +328,14 @@ nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store
NIXC_CATCH_ERRS
}
nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path)
{
if (context)
context->last_err_code = NIX_OK;
try {
return new nix_derivation{store->ptr->derivationFromPath(path->path)};
}
NIXC_CATCH_ERRS_NULL
}
} // extern "C"

19
src/libstore-c/nix_api_store.h
View file

@ -106,7 +106,7 @@ nix_err
nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data);
/**
* @brief Parse a Nix store path into a StorePath
* @brief Parse a Nix store path that includes the store dir into a StorePath
*
* @note Don't forget to free this path using nix_store_path_free()!
* @param[out] context Optional, stores error information
@ -188,9 +188,16 @@ nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_cal
/**
* @brief Create a `nix_derivation` from a JSON representation of that derivation.
*
* @note Unlike `nix_derivation_to_json`, this needs a `Store`. This is because
* over time we expect the internal representation of derivations in Nix to
* differ from accepted derivation formats. The store argument is here to help
* any logic needed to convert from JSON to the internal representation, in
* excess of just parsing.
*
* @param[out] context Optional, stores error information.
* @param[in] store nix store reference.
* @param[in] json JSON of the derivation as a string.
* @return A new derivation, or NULL on error. Free with `nix_derivation_free` when done using the `nix_derivation`.
*/
nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json);
@ -242,6 +249,16 @@ nix_err nix_store_get_fs_closure(
void * userdata,
void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path));
/**
* @brief Returns the derivation associated with the store path
*
* @param[out] context Optional, stores error information
* @param[in] store The nix store
* @param[in] path The nix store path
* @return A new derivation, or NULL on error. Free with `nix_derivation_free` when done using the `nix_derivation`.
*/
nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path);
// cffi end
#ifdef __cplusplus
}

19
src/libstore-c/nix_api_store/derivation.h
View file

@ -20,6 +20,14 @@ extern "C" {
/** @brief Nix Derivation */
typedef struct nix_derivation nix_derivation;
/**
* @brief Copy a `nix_derivation`
*
* @param[in] d the derivation to copy
* @return a new `nix_derivation`
*/
nix_derivation * nix_derivation_clone(const nix_derivation * d);
/**
* @brief Deallocate a `nix_derivation`
*
@ -28,6 +36,17 @@ typedef struct nix_derivation nix_derivation;
*/
void nix_derivation_free(nix_derivation * drv);
/**
* @brief Gets the derivation as a JSON string
*
* @param[out] context Optional, stores error information
* @param[in] drv The derivation
* @param[in] callback Called with the JSON string
* @param[in] userdata Arbitrary data passed to the callback
*/
nix_err nix_derivation_to_json(
nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata);
// cffi end
#ifdef __cplusplus
}

42
src/libstore-c/nix_api_store/store_path.h
View file

@ -10,6 +10,9 @@
* @brief Store path operations
*/
#include <stddef.h>
#include <stdint.h>
#include "nix_api_util.h"
#ifdef __cplusplus
@ -44,6 +47,45 @@ void nix_store_path_free(StorePath * p);
*/
void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
/**
* @brief A store path hash
*
* Once decoded from "nix32" encoding, a store path hash is 20 raw bytes.
*/
typedef struct nix_store_path_hash_part
{
uint8_t bytes[20];
} nix_store_path_hash_part;
/**
* @brief Get the path hash (e.g. "<hash>" in /nix/store/<hash>-<name>)
*
* The hash is returned as raw bytes, decoded from "nix32" encoding.
*
* @param[out] context Optional, stores error information
* @param[in] store_path the path to get the hash from
* @param[out] hash_part_out the decoded hash as 20 raw bytes
* @return NIX_OK on success, error code on failure
*/
nix_err
nix_store_path_hash(nix_c_context * context, const StorePath * store_path, nix_store_path_hash_part * hash_part_out);
/**
* @brief Create a StorePath from its constituent parts (hash and name)
*
* This function constructs a store path from a hash and name, without needing
* a Store reference or the store directory prefix.
*
* @note Don't forget to free this path using nix_store_path_free()!
* @param[out] context Optional, stores error information
* @param[in] hash The store path hash (20 raw bytes)
* @param[in] name The store path name (the part after the hash)
* @param[in] name_len Length of the name string
* @return owned store path, NULL on error
*/
StorePath * nix_store_create_from_parts(
nix_c_context * context, const nix_store_path_hash_part * hash, const char name[/*name_len*/], size_t name_len);
// cffi end
#ifdef __cplusplus
}

3
src/libstore-test-support/include/nix/store/tests/nix_api_store.hh
View file

@ -50,7 +50,8 @@ protected:
#else
// resolve any symlinks in i.e. on macOS /tmp -> /private/tmp
// because this is not allowed for a nix store.
auto tmpl = nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", true);
auto tmpl =
nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", std::nullopt, true);
nixDir = mkdtemp((char *) tmpl.c_str());
#endif

27
src/libstore-tests/data/derivation/invariants/bad-depends-on-drv-pre.json Normal file
View file

@ -0,0 +1,27 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "InputAddressed throws when should be deferred",
"out": ""
},
"inputs": {
"drvs": {
"lg4c4b8r9hlczwprl6kgnzfd9mc1xmkk-dependency.drv": {
"dynamicOutputs": {},
"outputs": [
"out"
]
}
},
"srcs": []
},
"name": "depends-on-drv",
"outputs": {
"out": {
"path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
}
},
"system": "x86_64-linux",
"version": 4
}

18
src/libstore-tests/data/derivation/invariants/bad-env-var.json Normal file
View file

@ -0,0 +1,18 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Wrong env var value throws error",
"out": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "bad-env-var",
"outputs": {
"out": {}
},
"system": "x86_64-linux",
"version": 4
}

20
src/libstore-tests/data/derivation/invariants/bad-path.json Normal file
View file

@ -0,0 +1,20 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Wrong InputAddressed path throws error",
"out": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "bad-path",
"outputs": {
"out": {
"path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
}
},
"system": "x86_64-linux",
"version": 4
}

25
src/libstore-tests/data/derivation/invariants/depends-on-drv-pre.json Normal file
View file

@ -0,0 +1,25 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Deferred stays deferred with CA dependencies",
"out": ""
},
"inputs": {
"drvs": {
"lg4c4b8r9hlczwprl6kgnzfd9mc1xmkk-dependency.drv": {
"dynamicOutputs": {},
"outputs": [
"out"
]
}
},
"srcs": []
},
"name": "depends-on-drv",
"outputs": {
"out": {}
},
"system": "x86_64-linux",
"version": 4
}

20
src/libstore-tests/data/derivation/invariants/filled-in-deferred-empty-env-var-post.json Normal file
View file

@ -0,0 +1,20 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Fill in deferred output with empty env var",
"out": "/nix/store/bilpz1nq8qi9r3bzsp72n34yjgqg43ws-filled-in-deferred-empty-env-var"
},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "filled-in-deferred-empty-env-var",
"outputs": {
"out": {
"path": "bilpz1nq8qi9r3bzsp72n34yjgqg43ws-filled-in-deferred-empty-env-var"
}
},
"system": "x86_64-linux",
"version": 4
}

18
src/libstore-tests/data/derivation/invariants/filled-in-deferred-empty-env-var-pre.json Normal file
View file

@ -0,0 +1,18 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Fill in deferred output with empty env var",
"out": ""
},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "filled-in-deferred-empty-env-var",
"outputs": {
"out": {}
},
"system": "x86_64-linux",
"version": 4
}

20
src/libstore-tests/data/derivation/invariants/filled-in-deferred-no-env-var-post.json Normal file
View file

@ -0,0 +1,20 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Fill in deferred with missing env var",
"out": "/nix/store/wpk9qrgg77fyswhailap0gicgw98izx9-filled-in-deferred-no-env-var"
},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "filled-in-deferred-no-env-var",
"outputs": {
"out": {
"path": "wpk9qrgg77fyswhailap0gicgw98izx9-filled-in-deferred-no-env-var"
}
},
"system": "x86_64-linux",
"version": 4
}

17
src/libstore-tests/data/derivation/invariants/filled-in-deferred-no-env-var-pre.json Normal file
View file

@ -0,0 +1,17 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Fill in deferred with missing env var"
},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "filled-in-deferred-no-env-var",
"outputs": {
"out": {}
},
"system": "x86_64-linux",
"version": 4
}

20
src/libstore-tests/data/derivation/invariants/filled-in-idempotent.json Normal file
View file

@ -0,0 +1,20 @@
{
"args": [],
"builder": "/bin/sh",
"env": {
"__doc": "Correct path stays unchanged",
"out": "/nix/store/w4bk7hpyxzgy2gx8fsa8f952435pll3i-filled-in-already"
},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "filled-in-already",
"outputs": {
"out": {
"path": "w4bk7hpyxzgy2gx8fsa8f952435pll3i-filled-in-already"
}
},
"system": "x86_64-linux",
"version": 4
}

8
src/libstore-tests/data/dummy-store/empty.json Normal file
View file

@ -0,0 +1,8 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {},
"derivations": {}
}

22
src/libstore-tests/data/dummy-store/one-derivation.json Normal file
View file

@ -0,0 +1,22 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {},
"derivations": {
"rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "foo",
"outputs": {},
"system": "",
"version": 4
}
}
}

38
src/libstore-tests/data/dummy-store/one-flat-file.json Normal file
View file

@ -0,0 +1,38 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {
"5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": {
"contents": {
"contents": "asdf",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": {
"algorithm": "sha256",
"format": "base64",
"hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
},
"method": "nar"
},
"deriver": null,
"narHash": {
"algorithm": "sha256",
"format": "base64",
"hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
},
"narSize": 120,
"references": [],
"registrationTime": null,
"signatures": [],
"ultimate": false,
"version": 2
}
}
},
"derivations": {}
}

16
src/libstore-tests/data/dummy-store/one-realisation.json Normal file
View file

@ -0,0 +1,16 @@
{
"buildTrace": {
"ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": {
"out": {
"dependentRealisations": {},
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": []
}
}
},
"config": {
"store": "/nix/store"
},
"contents": {},
"derivations": {}
}

45
src/libstore-tests/derivation.cc → src/libstore-tests/derivation/external-formats.cc
View file

@ -1,57 +1,14 @@
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include "nix/util/experimental-features.hh"
#include "nix/store/derivations.hh"
#include "nix/store/tests/libstore.hh"
#include "derivation/test-support.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
using nlohmann::json;
class DerivationTest : public virtual CharacterizationTest, public LibStoreTest
{
std::filesystem::path unitTestData = getUnitTestData() / "derivation";
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
};
class CaDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "ca-derivations");
}
};
class DynDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
}
};
class ImpureDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "impure-derivations");
}
};
TEST_F(DerivationTest, BadATerm_version)
{
ASSERT_THROW(

264
src/libstore-tests/derivation/invariants.cc Normal file
View file

@ -0,0 +1,264 @@
#include <gtest/gtest.h>
#include <nlohmann/json.hpp>
#include "nix/store/derivations.hh"
#include "nix/store/tests/libstore.hh"
#include "nix/store/dummy-store-impl.hh"
#include "nix/util/tests/json-characterization.hh"
#include "derivation/test-support.hh"
namespace nix {
class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTest<Derivation>
{
std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "invariants";
protected:
FillInOutputPathsTest()
: LibStoreTest([]() {
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
config->readOnly = false;
return config->openDummyStore();
}())
{
}
/**
* Create a CA floating output derivation and write it to the store.
* This is useful for creating dependencies that will cause downstream
* derivations to remain deferred.
*/
StorePath makeCAFloatingDependency(std::string_view name)
{
Derivation depDrv;
depDrv.name = name;
depDrv.platform = "x86_64-linux";
depDrv.builder = "/bin/sh";
depDrv.outputs = {
{
"out",
// will ensure that downstream is deferred
DerivationOutput{DerivationOutput::CAFloating{
.method = ContentAddressMethod::Raw::NixArchive,
.hashAlgo = HashAlgorithm::SHA256,
}},
},
};
depDrv.env = {{"out", ""}};
// Fill in the dependency derivation's output paths
depDrv.fillInOutputPaths(*store);
// Write the dependency to the store
return writeDerivation(*store, depDrv, NoRepair);
}
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
};
TEST_F(FillInOutputPathsTest, fillsDeferredOutputs_emptyStringEnvVar)
{
using nlohmann::json;
// Before: Derivation with deferred output
Derivation drv;
drv.name = "filled-in-deferred-empty-env-var";
drv.platform = "x86_64-linux";
drv.builder = "/bin/sh";
drv.outputs = {
{"out", DerivationOutput{DerivationOutput::Deferred{}}},
};
drv.env = {{"__doc", "Fill in deferred output with empty env var"}, {"out", ""}};
// Serialize before state
checkpointJson("filled-in-deferred-empty-env-var-pre", drv);
drv.fillInOutputPaths(*store);
// Serialize after state
checkpointJson("filled-in-deferred-empty-env-var-post", drv);
// After: Should have been converted to InputAddressed
auto * outputP = std::get_if<DerivationOutput::InputAddressed>(&drv.outputs.at("out").raw);
ASSERT_TRUE(outputP);
auto & output = *outputP;
// Environment variable should be filled in
EXPECT_EQ(drv.env.at("out"), store->printStorePath(output.path));
}
TEST_F(FillInOutputPathsTest, fillsDeferredOutputs_empty_string_var)
{
using nlohmann::json;
// Before: Derivation with deferred output
Derivation drv;
drv.name = "filled-in-deferred-no-env-var";
drv.platform = "x86_64-linux";
drv.builder = "/bin/sh";
drv.outputs = {
{"out", DerivationOutput{DerivationOutput::Deferred{}}},
};
drv.env = {
{"__doc", "Fill in deferred with missing env var"},
};
// Serialize before state
checkpointJson("filled-in-deferred-no-env-var-pre", drv);
drv.fillInOutputPaths(*store);
// Serialize after state
checkpointJson("filled-in-deferred-no-env-var-post", drv);
// After: Should have been converted to InputAddressed
auto * outputP = std::get_if<DerivationOutput::InputAddressed>(&drv.outputs.at("out").raw);
ASSERT_TRUE(outputP);
auto & output = *outputP;
// Environment variable should be filled in
EXPECT_EQ(drv.env.at("out"), store->printStorePath(output.path));
}
TEST_F(FillInOutputPathsTest, preservesInputAddressedOutputs)
{
auto expectedPath = StorePath{"w4bk7hpyxzgy2gx8fsa8f952435pll3i-filled-in-already"};
Derivation drv;
drv.name = "filled-in-already";
drv.platform = "x86_64-linux";
drv.builder = "/bin/sh";
drv.outputs = {
{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = expectedPath}}},
};
drv.env = {
{"__doc", "Correct path stays unchanged"},
{"out", store->printStorePath(expectedPath)},
};
// Serialize before state
checkpointJson("filled-in-idempotent", drv);
auto drvBefore = drv;
drv.fillInOutputPaths(*store);
// Should still be no change
EXPECT_EQ(drv, drvBefore);
}
TEST_F(FillInOutputPathsTest, throwsOnIncorrectInputAddressedPath)
{
auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"};
Derivation drv;
drv.name = "bad-path";
drv.platform = "x86_64-linux";
drv.builder = "/bin/sh";
drv.outputs = {
{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}},
};
drv.env = {
{"__doc", "Wrong InputAddressed path throws error"},
{"out", store->printStorePath(wrongPath)},
};
// Serialize before state
checkpointJson("bad-path", drv);
ASSERT_THROW(drv.fillInOutputPaths(*store), Error);
}
TEST_F(FillInOutputPathsTest, throwsOnIncorrectEnvVar)
{
auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"};
Derivation drv;
drv.name = "bad-env-var";
drv.platform = "x86_64-linux";
drv.builder = "/bin/sh";
drv.outputs = {
{"out", DerivationOutput{DerivationOutput::Deferred{}}},
};
drv.env = {
{"__doc", "Wrong env var value throws error"},
{"out", store->printStorePath(wrongPath)},
};
// Serialize before state
checkpointJson("bad-env-var", drv);
ASSERT_THROW(drv.fillInOutputPaths(*store), Error);
}
TEST_F(FillInOutputPathsTest, preservesDeferredWithInputDrvs)
{
using nlohmann::json;
// Create a CA floating dependency derivation
auto depDrvPath = makeCAFloatingDependency("dependency");
// Create a derivation that depends on the dependency
Derivation drv;
drv.name = "depends-on-drv";
drv.platform = "x86_64-linux";
drv.builder = "/bin/sh";
drv.outputs = {
{"out", DerivationOutput{DerivationOutput::Deferred{}}},
};
drv.env = {
{"__doc", "Deferred stays deferred with CA dependencies"},
{"out", ""},
};
// Add the real input derivation dependency
drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
// Serialize before state
checkpointJson("depends-on-drv-pre", drv);
auto drvBefore = drv;
// Apply fillInOutputPaths
drv.fillInOutputPaths(*store);
// Derivation should be unchanged
EXPECT_EQ(drv, drvBefore);
}
TEST_F(FillInOutputPathsTest, throwsOnPatWhenShouldBeDeffered)
{
using nlohmann::json;
// Create a CA floating dependency derivation
auto depDrvPath = makeCAFloatingDependency("dependency");
auto wrongPath = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"};
// Create a derivation that depends on the dependency
Derivation drv;
drv.name = "depends-on-drv";
drv.platform = "x86_64-linux";
drv.builder = "/bin/sh";
drv.outputs = {
{"out", DerivationOutput{DerivationOutput::InputAddressed{.path = wrongPath}}},
};
drv.env = {
{"__doc", "InputAddressed throws when should be deferred"},
{"out", ""},
};
// Add the real input derivation dependency
drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
// Serialize before state
checkpointJson("bad-depends-on-drv-pre", drv);
// Apply fillInOutputPaths
ASSERT_THROW(drv.fillInOutputPaths(*store), Error);
}
} // namespace nix

52
src/libstore-tests/derivation/test-support.hh Normal file
View file

@ -0,0 +1,52 @@
#pragma once
#include <gtest/gtest.h>
#include "nix/util/experimental-features.hh"
#include "nix/store/tests/libstore.hh"
#include "nix/util/tests/characterization.hh"
namespace nix {
class DerivationTest : public virtual CharacterizationTest, public LibStoreTest
{
std::filesystem::path unitTestData = getUnitTestData() / "derivation";
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
};
class CaDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "ca-derivations");
}
};
class DynDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
}
};
class ImpureDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "impure-derivations");
}
};
} // namespace nix

115
src/libstore-tests/dummy-store.cc
View file

@ -1,11 +1,32 @@
#include <gtest/gtest.h>
#include <nlohmann/json.hpp>
#include "nix/util/memory-source-accessor.hh"
#include "nix/store/dummy-store-impl.hh"
#include "nix/store/globals.hh"
#include "nix/store/realisation.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
class DummyStoreTest : public virtual CharacterizationTest
{
std::filesystem::path unitTestData = getUnitTestData() / "dummy-store";
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
static void SetUpTestSuite()
{
initLibStore(false);
}
};
TEST(DummyStore, realisation_read)
{
initLibStore(/*loadConfig=*/false);
@ -27,7 +48,7 @@ TEST(DummyStore, realisation_read)
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
};
store->buildTrace.insert({drvHash, {{outputName, make_ref<UnkeyedRealisation>(value)}}});
store->buildTrace.insert({drvHash, {{outputName, value}}});
auto value2 = store->queryRealisation({drvHash, outputName});
@ -35,4 +56,96 @@ TEST(DummyStore, realisation_read)
EXPECT_EQ(*value2, value);
}
/* ----------------------------------------------------------------------------
* JSON
* --------------------------------------------------------------------------*/
using nlohmann::json;
struct DummyStoreJsonTest : DummyStoreTest,
JsonCharacterizationTest<ref<DummyStore>>,
::testing::WithParamInterface<std::pair<std::string_view, ref<DummyStore>>>
{};
TEST_P(DummyStoreJsonTest, from_json)
{
auto & [name, expected] = GetParam();
using namespace nlohmann;
/* Cannot use `readJsonTest` because need to dereference the stores
for equality. */
readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
auto encoded = json::parse(encodedRaw);
ref<DummyStore> decoded = adl_serializer<ref<DummyStore>>::from_json(encoded);
ASSERT_EQ(*decoded, *expected);
});
}
TEST_P(DummyStoreJsonTest, to_json)
{
auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] {
initLibStore(false);
auto writeCfg = make_ref<DummyStore::Config>(DummyStore::Config::Params{});
writeCfg->readOnly = false;
return ::testing::Values(
std::pair{
"empty",
make_ref<DummyStore::Config>(DummyStore::Config::Params{})->openDummyStore(),
},
std::pair{
"one-flat-file",
[&] {
auto store = writeCfg->openDummyStore();
store->addToStore(
"my-file",
SourcePath{
[] {
auto sc = make_ref<MemorySourceAccessor>();
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents = "asdf",
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256);
return store;
}(),
},
std::pair{
"one-derivation",
[&] {
auto store = writeCfg->openDummyStore();
Derivation drv;
drv.name = "foo";
store->writeDerivation(drv);
return store;
}(),
},
std::pair{
"one-realisation",
[&] {
auto store = writeCfg->openDummyStore();
store->buildTrace.insert_or_assign(
Hash::parseExplicitFormatUnprefixed(
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
HashAlgorithm::SHA256,
HashFormat::Base16),
std::map<std::string, UnkeyedRealisation>{
{
"out",
UnkeyedRealisation{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
});
return store;
}(),
});
}());
} // namespace nix

3
src/libstore-tests/meson.build
View file

@ -58,7 +58,8 @@ sources = files(
'common-protocol.cc',
'content-address.cc',
'derivation-advanced-attrs.cc',
'derivation.cc',
'derivation/external-formats.cc',
'derivation/invariants.cc',
'derived-path.cc',
'downstream-placeholder.cc',
'dummy-store.cc',

168
src/libstore-tests/nix_api_store.cc
View file

@ -1,5 +1,7 @@
#include <fstream>
#include <nlohmann/json.hpp>
#include "nix_api_util.h"
#include "nix_api_store.h"
@ -92,6 +94,70 @@ TEST_F(nix_api_store_test, DoesNotCrashWhenContextIsNull)
nix_store_path_free(path);
}
// Verify it's 20 bytes
static_assert(sizeof(nix_store_path_hash_part::bytes) == 20);
static_assert(sizeof(nix_store_path_hash_part::bytes) == sizeof(nix_store_path_hash_part));
TEST_F(nix_api_store_test, nix_store_path_hash)
{
StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str());
ASSERT_NE(path, nullptr);
nix_store_path_hash_part hash;
auto ret = nix_store_path_hash(ctx, path, &hash);
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
// The hash should be non-zero
bool allZero = true;
for (size_t i = 0; i < sizeof(hash.bytes); i++) {
if (hash.bytes[i] != 0) {
allZero = false;
break;
}
}
ASSERT_FALSE(allZero);
nix_store_path_free(path);
}
TEST_F(nix_api_store_test, nix_store_create_from_parts_roundtrip)
{
// Parse a path
StorePath * original = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str());
EXPECT_NE(original, nullptr);
// Get its hash
nix_store_path_hash_part hash;
auto ret = nix_store_path_hash(ctx, original, &hash);
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
// Get its name
std::string name;
nix_store_path_name(original, OBSERVE_STRING(name));
// Reconstruct from parts
StorePath * reconstructed = nix_store_create_from_parts(ctx, &hash, name.c_str(), name.size());
assert_ctx_ok();
ASSERT_NE(reconstructed, nullptr);
// Should be equal
EXPECT_EQ(original->path, reconstructed->path);
nix_store_path_free(original);
nix_store_path_free(reconstructed);
}
TEST_F(nix_api_store_test, nix_store_create_from_parts_invalid_name)
{
nix_store_path_hash_part hash = {};
// Invalid name with spaces
StorePath * path = nix_store_create_from_parts(ctx, &hash, "invalid name", 12);
ASSERT_EQ(path, nullptr);
ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
}
TEST_F(nix_api_store_test, get_version)
{
std::string str;
@ -146,9 +212,9 @@ TEST_F(nix_api_store_test, nix_store_real_path)
TEST_F(nix_api_util_context, nix_store_real_path_relocated)
{
auto tmp = nix::createTempDir();
std::string storeRoot = tmp + "/store";
std::string stateDir = tmp + "/state";
std::string logDir = tmp + "/log";
std::string storeRoot = tmp / "store";
std::string stateDir = tmp / "state";
std::string logDir = tmp / "log";
const char * rootkv[] = {"root", storeRoot.c_str()};
const char * statekv[] = {"state", stateDir.c_str()};
const char * logkv[] = {"log", logDir.c_str()};
@ -184,7 +250,8 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated)
TEST_F(nix_api_util_context, nix_store_real_path_binary_cache)
{
Store * store = nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir()).c_str(), nullptr);
Store * store =
nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir().string()).c_str(), nullptr);
assert_ctx_ok();
ASSERT_NE(store, nullptr);
@ -795,4 +862,97 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagati
ASSERT_EQ(call_count, 1); // Should have been called exactly once, then aborted
}
/**
* @brief Helper function to load JSON from a test data file
*
* @param filename Relative path from _NIX_TEST_UNIT_DATA
* @return JSON string contents of the file
*/
static std::string load_json_from_test_data(const char * filename)
{
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
std::ifstream t{unitTestData / filename};
std::stringstream buffer;
buffer << t.rdbuf();
return buffer.str();
}
TEST_F(nix_api_store_test, nix_derivation_to_json_roundtrip)
{
// Load JSON from test data
auto originalJson = load_json_from_test_data("derivation/invariants/filled-in-deferred-empty-env-var-pre.json");
// Parse to derivation
auto * drv = nix_derivation_from_json(ctx, store, originalJson.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
// Convert back to JSON
std::string convertedJson;
auto ret = nix_derivation_to_json(ctx, drv, OBSERVE_STRING(convertedJson));
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
ASSERT_FALSE(convertedJson.empty());
// Parse both JSON strings to compare (ignoring whitespace differences)
auto originalParsed = nlohmann::json::parse(originalJson);
auto convertedParsed = nlohmann::json::parse(convertedJson);
// Remove parts that will be different due to filling-in.
originalParsed.at("outputs").erase("out");
originalParsed.at("env").erase("out");
convertedParsed.at("outputs").erase("out");
convertedParsed.at("env").erase("out");
// They should be equivalent
ASSERT_EQ(originalParsed, convertedParsed);
nix_derivation_free(drv);
}
TEST_F(nix_api_store_test, nix_derivation_store_round_trip)
{
// Load a derivation from JSON
auto json = load_json_from_test_data("derivation/invariants/filled-in-deferred-empty-env-var-pre.json");
auto * drv = nix_derivation_from_json(ctx, store, json.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
// Add to store
auto * drvPath = nix_add_derivation(ctx, store, drv);
assert_ctx_ok();
ASSERT_NE(drvPath, nullptr);
// Retrieve from store
auto * drv2 = nix_store_drv_from_store_path(ctx, store, drvPath);
assert_ctx_ok();
ASSERT_NE(drv2, nullptr);
// The round trip should make the same derivation
ASSERT_EQ(drv->drv, drv2->drv);
nix_store_path_free(drvPath);
nix_derivation_free(drv);
nix_derivation_free(drv2);
}
TEST_F(nix_api_store_test, nix_derivation_clone)
{
// Load a derivation from JSON
auto json = load_json_from_test_data("derivation/invariants/filled-in-deferred-empty-env-var-pre.json");
auto * drv = nix_derivation_from_json(ctx, store, json.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
// Clone the derivation
auto * drv2 = nix_derivation_clone(drv);
ASSERT_NE(drv2, nullptr);
// The clone should be equal
ASSERT_EQ(drv->drv, drv2->drv);
nix_derivation_free(drv);
nix_derivation_free(drv2);
}
} // namespace nixC

13
src/libstore/aws-creds.cc
View file

@ -79,7 +79,18 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
public:
AwsCredentialProviderImpl()
{
apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast<FILE *>(nullptr));
// Map Nix's verbosity to AWS CRT log level
Aws::Crt::LogLevel logLevel;
if (verbosity >= lvlVomit) {
logLevel = Aws::Crt::LogLevel::Trace;
} else if (verbosity >= lvlDebug) {
logLevel = Aws::Crt::LogLevel::Debug;
} else if (verbosity >= lvlChatty) {
logLevel = Aws::Crt::LogLevel::Info;
} else {
logLevel = Aws::Crt::LogLevel::Warn;
}
apiHandle.InitializeLogging(logLevel, stderr);
}
AwsCredentials getCredentialsRaw(const std::string & profile);

20
src/libstore/binary-cache-store.cc
View file

@ -418,10 +418,20 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
{
auto info = queryPathInfo(storePath).cast<const NarInfo>();
LengthSink narSize;
TeeSink tee{sink, narSize};
uint64_t narSize = 0;
auto decompressor = makeDecompressionSink(info->compression, tee);
LambdaSink uncompressedSink{
[&](std::string_view data) {
narSize += data.size();
sink(data);
},
[&]() {
stats.narRead++;
// stats.narReadCompressedBytes += nar->size(); // FIXME
stats.narReadBytes += narSize;
}};
auto decompressor = makeDecompressionSink(info->compression, uncompressedSink);
try {
getFile(info->url, *decompressor);
@ -431,9 +441,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
decompressor->finish();
stats.narRead++;
// stats.narReadCompressedBytes += nar->size(); // FIXME
stats.narReadBytes += narSize.length;
// Note: don't do anything here because it's never reached if we're called as a coroutine.
}
void BinaryCacheStore::queryPathInfoUncached(

2
src/libstore/build/derivation-building-goal.cc
View file

@ -307,7 +307,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
crashes. If we can't acquire the lock, then continue; hopefully some
other goal can start a build, and if not, the main loop will sleep a few
seconds and then retry this goal. */
PathSet lockFiles;
std::set<std::filesystem::path> lockFiles;
/* FIXME: Should lock something like the drv itself so we don't build same
CA drv concurrently */
if (auto * localStore = dynamic_cast<LocalStore *>(&worker.store)) {

4
src/libstore/daemon.cc
View file

@ -964,7 +964,7 @@ static void performOp(
case WorkerProto::Op::RegisterDrvOutput: {
logger->startWork();
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) {
auto outputId = DrvOutput::parse(readString(conn.from));
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
auto outputPath = StorePath(readString(conn.from));
store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId});
} else {
@ -977,7 +977,7 @@ static void performOp(
case WorkerProto::Op::QueryRealisation: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(conn.from));
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
auto info = store->queryRealisation(outputId);
logger->stopWork();
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) {

218
src/libstore/derivations.cc
View file

@ -1203,6 +1203,136 @@ std::optional<BasicDerivation> Derivation::tryResolve(
return resolved;
}
/**
* Process `InputAddressed`, `Deferred`, and `CAFixed` outputs.
*
* For `InputAddressed` outputs or `Deferred` outputs:
*
* - with `Regular` hash kind, validate `InputAddressed` outputs have
* the correct path (throws if mismatch). For `Deferred` outputs:
* - if `fillIn` is true, fill in the output path to make `InputAddressed`
* - if `fillIn` is false, throw an error
* Then validate or fill in the environment variable with the path.
*
* - with `Deferred` hash kind, validate that the output is either
* `InputAddressed` (error) or `Deferred` (correct).
*
* For `CAFixed` outputs, validate or fill in the environment variable
* with the computed path.
*
* @tparam fillIn If true, fill in missing output paths and environment
* variables. If false, validate that all paths are correct (throws on
* mismatch).
*/
template<bool fillIn>
static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName)
{
std::optional<DrvHash> hashesModulo;
for (auto & [outputName, output] : drv.outputs) {
auto envHasRightPath = [&](const StorePath & actual) {
if constexpr (fillIn) {
auto j = drv.env.find(outputName);
/* Fill in mode: fill in missing or empty environment
variables */
if (j == drv.env.end())
drv.env.insert(j, {outputName, store.printStorePath(actual)});
else if (j->second == "")
j->second = store.printStorePath(actual);
/* We know validation will succeed after fill-in, but
just to be extra sure, validate unconditionally */
}
auto j = drv.env.find(outputName);
if (j == drv.env.end())
throw Error(
"derivation has missing environment variable '%s', should be '%s' but is not present",
outputName,
store.printStorePath(actual));
if (j->second != store.printStorePath(actual))
throw Error(
"derivation has incorrect environment variable '%s', should be '%s' but is actually '%s'",
outputName,
store.printStorePath(actual),
j->second);
};
auto hash = [&]<typename Output>(const Output & outputVariant) {
if (!hashesModulo) {
// somewhat expensive so we do lazily
hashesModulo = hashDerivationModulo(store, drv, true);
}
switch (hashesModulo->kind) {
case DrvHash::Kind::Regular: {
auto h = get(hashesModulo->hashes, outputName);
if (!h)
throw Error("derivation produced no hash for output '%s'", outputName);
auto outPath = store.makeOutputPath(outputName, *h, drvName);
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
if (outputVariant.path == outPath) {
return; // Correct case
}
/* Error case, an explicitly wrong path is
always an error. */
throw Error(
"derivation has incorrect output '%s', should be '%s'",
store.printStorePath(outputVariant.path),
store.printStorePath(outPath));
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
if constexpr (fillIn)
/* Fill in output path for Deferred
outputs */
output = DerivationOutput::InputAddressed{
.path = outPath,
};
else
/* Validation mode: deferred outputs
should have been filled in */
throw Error(
"derivation has incorrect deferred output, should be '%s'", store.printStorePath(outPath));
} else {
/* Will never happen, based on where
`hash` is called. */
static_assert(false);
}
envHasRightPath(outPath);
break;
}
case DrvHash::Kind::Deferred:
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
/* Error case, an explicitly wrong path is
always an error. */
throw Error(
"derivation has incorrect output '%s', should be deferred",
store.printStorePath(outputVariant.path));
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
/* Correct: Deferred output with Deferred
hash kind. */
} else {
/* Will never happen, based on where
`hash` is called. */
static_assert(false);
}
break;
}
};
std::visit(
overloaded{
[&](const DerivationOutput::InputAddressed & o) { hash(o); },
[&](const DerivationOutput::Deferred & o) { hash(o); },
[&](const DerivationOutput::CAFixed & dof) { envHasRightPath(dof.path(store, drvName, outputName)); },
[&](const auto &) {
// Nothing to do for other output types
},
},
output.raw);
}
/* Don't need the answer, but do this anyways to assert is proper
combination. The code above is more general and naturally allows
combinations that are currently prohibited. */
drv.type();
}
void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
{
assert(drvPath.isDerivation());
@ -1210,67 +1340,43 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
drvName = drvName.substr(0, drvName.size() - drvExtension.size());
if (drvName != name) {
throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name);
throw Error("derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name);
}
auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) {
auto j = env.find(varName);
if (j == env.end() || store.parseStorePath(j->second) != actual)
throw Error(
"derivation '%s' has incorrect environment variable '%s', should be '%s'",
store.printStorePath(drvPath),
varName,
store.printStorePath(actual));
};
// Don't need the answer, but do this anyways to assert is proper
// combination. The code below is more general and naturally allows
// combinations that are currently prohibited.
type();
std::optional<DrvHash> hashesModulo;
for (auto & i : outputs) {
std::visit(
overloaded{
[&](const DerivationOutput::InputAddressed & doia) {
if (!hashesModulo) {
// somewhat expensive so we do lazily
hashesModulo = hashDerivationModulo(store, *this, true);
}
auto currentOutputHash = get(hashesModulo->hashes, i.first);
if (!currentOutputHash)
throw Error(
"derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
store.printStorePath(drvPath),
store.printStorePath(doia.path),
i.first);
StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName);
if (doia.path != recomputed)
throw Error(
"derivation '%s' has incorrect output '%s', should be '%s'",
store.printStorePath(drvPath),
store.printStorePath(doia.path),
store.printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
[&](const DerivationOutput::CAFixed & dof) {
auto path = dof.path(store, drvName, i.first);
envHasRightPath(path, i.first);
},
[&](const DerivationOutput::CAFloating &) {
/* Nothing to check */
},
[&](const DerivationOutput::Deferred &) {
/* Nothing to check */
},
[&](const DerivationOutput::Impure &) {
/* Nothing to check */
},
},
i.second.raw);
try {
checkInvariants(store);
} catch (Error & e) {
e.addTrace({}, "while checking derivation '%s'", store.printStorePath(drvPath));
throw;
}
}
void Derivation::checkInvariants(Store & store) const
{
processDerivationOutputPaths<false>(store, *this, name);
}
void Derivation::fillInOutputPaths(Store & store)
{
processDerivationOutputPaths<true>(store, *this, name);
}
Derivation Derivation::parseJsonAndValidate(Store & store, const nlohmann::json & json)
{
auto drv = static_cast<Derivation>(json);
drv.fillInOutputPaths(store);
try {
drv.checkInvariants(store);
} catch (Error & e) {
e.addTrace({}, "while checking derivation from JSON with name '%s'", drv.name);
throw;
}
return drv;
}
const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure");
} // namespace nix

115
src/libstore/dummy-store.cc
View file

@ -2,6 +2,7 @@
#include "nix/util/archive.hh"
#include "nix/util/callback.hh"
#include "nix/util/memory-source-accessor.hh"
#include "nix/util/json-utils.hh"
#include "nix/store/dummy-store-impl.hh"
#include "nix/store/realisation.hh"
@ -16,6 +17,16 @@ std::string DummyStoreConfig::doc()
;
}
bool DummyStore::PathInfoAndContents::operator==(const PathInfoAndContents & other) const
{
return info == other.info && contents->root == other.contents->root;
}
bool DummyStore::operator==(const DummyStore & other) const
{
return contents == other.contents && derivations == other.derivations && buildTrace == other.buildTrace;
}
namespace {
class WholeStoreViewAccessor : public SourceAccessor
@ -320,9 +331,8 @@ struct DummyStoreImpl : DummyStore
void registerDrvOutput(const Realisation & output) override
{
auto ref = make_ref<UnkeyedRealisation>(output);
buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, ref}}}, [&](auto & kv) {
kv.second.insert_or_assign(output.id.outputName, make_ref<UnkeyedRealisation>(output));
buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) {
kv.second.insert_or_assign(output.id.outputName, output);
});
}
@ -333,7 +343,7 @@ struct DummyStoreImpl : DummyStore
buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) {
if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) {
visited = true;
callback(it->second.get_ptr());
callback(std::make_shared<UnkeyedRealisation>(it->second));
}
});
@ -377,3 +387,100 @@ ref<DummyStore> DummyStore::Config::openDummyStore() const
static RegisterStoreImplementation<DummyStore::Config> regDummyStore;
} // namespace nix
namespace nlohmann {
using namespace nix;
DummyStore::PathInfoAndContents adl_serializer<DummyStore::PathInfoAndContents>::from_json(const json & json)
{
auto & obj = getObject(json);
return DummyStore::PathInfoAndContents{
.info = valueAt(obj, "info"),
.contents = make_ref<MemorySourceAccessor>(valueAt(obj, "contents")),
};
}
void adl_serializer<DummyStore::PathInfoAndContents>::to_json(json & json, const DummyStore::PathInfoAndContents & val)
{
json = {
{"info", val.info},
{"contents", *val.contents},
};
}
ref<DummyStoreConfig> adl_serializer<ref<DummyStore::Config>>::from_json(const json & json)
{
auto & obj = getObject(json);
auto cfg = make_ref<DummyStore::Config>(DummyStore::Config::Params{});
const_cast<PathSetting &>(cfg->storeDir_).set(getString(valueAt(obj, "store")));
cfg->readOnly = true;
return cfg;
}
void adl_serializer<DummyStoreConfig>::to_json(json & json, const DummyStoreConfig & val)
{
json = {
{"store", val.storeDir},
};
}
ref<DummyStore> adl_serializer<ref<DummyStore>>::from_json(const json & json)
{
auto & obj = getObject(json);
ref<DummyStore> res = adl_serializer<ref<DummyStoreConfig>>::from_json(valueAt(obj, "config"))->openDummyStore();
for (auto & [k, v] : getObject(valueAt(obj, "contents")))
res->contents.insert({StorePath{k}, v});
for (auto & [k, v] : getObject(valueAt(obj, "derivations")))
res->derivations.insert({StorePath{k}, v});
for (auto & [k0, v] : getObject(valueAt(obj, "buildTrace"))) {
for (auto & [k1, v2] : getObject(v)) {
UnkeyedRealisation realisation = v2;
res->buildTrace.insert_or_visit(
{
Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64),
{{k1, realisation}},
},
[&](auto & kv) { kv.second.insert_or_assign(k1, realisation); });
}
}
return res;
}
void adl_serializer<DummyStore>::to_json(json & json, const DummyStore & val)
{
json = {
{"config", *val.config},
{"contents",
[&] {
auto obj = json::object();
val.contents.cvisit_all([&](const auto & kv) {
auto & [k, v] = kv;
obj[k.to_string()] = v;
});
return obj;
}()},
{"derivations",
[&] {
auto obj = json::object();
val.derivations.cvisit_all([&](const auto & kv) {
auto & [k, v] = kv;
obj[k.to_string()] = v;
});
return obj;
}()},
{"buildTrace",
[&] {
auto obj = json::object();
val.buildTrace.cvisit_all([&](const auto & kv) {
auto & [k, v] = kv;
auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object();
for (auto & [k2, v2] : kv.second)
obj2[k2] = v2;
});
return obj;
}()},
};
}
} // namespace nlohmann

2
src/libstore/globals.cc
View file

@ -486,7 +486,7 @@ void initLibStore(bool loadConfig)
/* On macOS, don't use the per-session TMPDIR (as set e.g. by
sshd). This breaks build users because they don't have access
to the TMPDIR, in particular in ‘nix-store --serve’. */
if (hasPrefix(defaultTempDir(), "/var/folders/"))
if (hasPrefix(defaultTempDir().string(), "/var/folders/"))
unsetenv("TMPDIR");
#endif

62
src/libstore/include/nix/store/derivations.hh
View file

@ -368,9 +368,48 @@ struct Derivation : BasicDerivation
* This is mainly a matter of checking the outputs, where our C++
* representation supports all sorts of combinations we do not yet
* allow.
*
* This overload does not validate the derivation name or add path
* context to errors. Use this when you don't have a `StorePath` or
* when you want to handle error context yourself.
*
* @param store The store to use for validation
*/
void checkInvariants(Store & store) const;
/**
* This overload does everything the base `checkInvariants` does,
* but also validates that the derivation name matches the path, and
* improves any error messages that occur using the derivation path.
*
* @param store The store to use for validation
* @param drvPath The path to this derivation
*/
void checkInvariants(Store & store, const StorePath & drvPath) const;
/**
* Fill in output paths as needed.
*
* For input-addressed derivations (ready or deferred), it computes
* the derivation hash modulo and based on the result:
*
* - If `Regular`: converts `Deferred` outputs to `InputAddressed`,
* and ensures all `InputAddressed` outputs (whether preexisting
* or newly computed) have the right computed paths. Likewise
* defines (if absent or the empty string) or checks (if
* preexisting and non-empty) environment variables for each
* output with their path.
*
* - If `Deferred`: converts `InputAddressed` to `Deferred`.
*
* Also for fixed-output content-addressed derivations, likewise
* updates output paths in env vars.
*
* @param store The store to use for path computation
* @param drvName The derivation name (without .drv extension)
*/
void fillInOutputPaths(Store & store);
Derivation() = default;
Derivation(const BasicDerivation & bd)
@ -383,6 +422,29 @@ struct Derivation : BasicDerivation
{
}
/**
* Parse a derivation from JSON, and also perform various
* conveniences such as:
*
* 1. Filling in output paths in as needed/required.
*
* 2. Checking invariants in general.
*
* In the future it might also do things like:
*
* - assist with the migration from older JSON formats.
*
* - (a somewhat example of the above) initialize
* `DerivationOptions` from their traditional encoding inside the
* `env` and `structuredAttrs`.
*
* @param store The store to use for path computation and validation
* @param json The JSON representation of the derivation
* @return A validated derivation with output paths filled in
* @throws Error if parsing fails, output paths can't be computed, or validation fails
*/
static Derivation parseJsonAndValidate(Store & store, const nlohmann::json & json);
bool operator==(const Derivation &) const = default;
// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
// auto operator <=> (const Derivation &) const = default;

12
src/libstore/include/nix/store/dummy-store-impl.hh
View file

@ -23,6 +23,8 @@ struct DummyStore : virtual Store
{
UnkeyedValidPathInfo info;
ref<MemorySourceAccessor> contents;
bool operator==(const PathInfoAndContents &) const;
};
/**
@ -47,13 +49,21 @@ struct DummyStore : virtual Store
* outer map for the derivation, and inner maps for the outputs of a
* given derivation.
*/
boost::concurrent_flat_map<Hash, std::map<std::string, ref<UnkeyedRealisation>>> buildTrace;
boost::concurrent_flat_map<Hash, std::map<std::string, UnkeyedRealisation>> buildTrace;
DummyStore(ref<const Config> config)
: Store{*config}
, config(config)
{
}
bool operator==(const DummyStore &) const;
};
template<>
struct json_avoids_null<DummyStore::PathInfoAndContents> : std::true_type
{};
} // namespace nix
JSON_IMPL(nix::DummyStore::PathInfoAndContents)

30
src/libstore/include/nix/store/dummy-store.hh
View file

@ -2,6 +2,7 @@
///@file
#include "nix/store/store-api.hh"
#include "nix/util/json-impls.hh"
#include <boost/unordered/concurrent_flat_map.hpp>
@ -65,4 +66,33 @@ struct DummyStoreConfig : public std::enable_shared_from_this<DummyStoreConfig>,
}
};
template<>
struct json_avoids_null<nix::DummyStoreConfig> : std::true_type
{};
template<>
struct json_avoids_null<ref<nix::DummyStoreConfig>> : std::true_type
{};
template<>
struct json_avoids_null<nix::DummyStore> : std::true_type
{};
template<>
struct json_avoids_null<ref<nix::DummyStore>> : std::true_type
{};
} // namespace nix
namespace nlohmann {
template<>
JSON_IMPL_INNER_TO(nix::DummyStoreConfig);
template<>
JSON_IMPL_INNER_FROM(nix::ref<nix::DummyStoreConfig>);
template<>
JSON_IMPL_INNER_TO(nix::DummyStore);
template<>
JSON_IMPL_INNER_FROM(nix::ref<nix::DummyStore>);
} // namespace nlohmann

4
src/libstore/include/nix/store/globals.hh
View file

@ -101,7 +101,7 @@ public:
/**
* The directory where system configuration files are stored.
*/
Path nixConfDir;
std::filesystem::path nixConfDir;
/**
* A list of user configuration files to load.
@ -292,7 +292,7 @@ public:
Setting<std::string> builders{
this,
"@" + nixConfDir + "/machines",
"@" + nixConfDir.string() + "/machines",
"builders",
R"(
A semicolon- or newline-separated list of build machines.

12
src/libstore/include/nix/store/pathlocks.hh
View file

@ -1,6 +1,8 @@
#pragma once
///@file
#include <filesystem>
#include "nix/util/file-descriptor.hh"
namespace nix {
@ -10,12 +12,12 @@ namespace nix {
* -1 is returned if create is false and the lock could not be opened
* because it doesn't exist. Any other error throws an exception.
*/
AutoCloseFD openLockFile(const Path & path, bool create);
AutoCloseFD openLockFile(const std::filesystem::path & path, bool create);
/**
* Delete an open lock file.
*/
void deleteLockFile(const Path & path, Descriptor desc);
void deleteLockFile(const std::filesystem::path & path, Descriptor desc);
enum LockType { ltRead, ltWrite, ltNone };
@ -24,14 +26,14 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait);
class PathLocks
{
private:
typedef std::pair<Descriptor, Path> FDPair;
typedef std::pair<Descriptor, std::filesystem::path> FDPair;
std::list<FDPair> fds;
bool deletePaths;
public:
PathLocks();
PathLocks(const PathSet & paths, const std::string & waitMsg = "");
bool lockPaths(const PathSet & _paths, const std::string & waitMsg = "", bool wait = true);
PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
~PathLocks();
void unlock();
void setDeletion(bool deletePaths);

44
src/libstore/include/nix/store/profiles.hh
View file

@ -7,12 +7,13 @@
* See the manual for additional information.
*/
#include "nix/util/types.hh"
#include "nix/store/pathlocks.hh"
#include <filesystem>
#include <optional>
#include <time.h>
#include "nix/util/types.hh"
#include "nix/store/pathlocks.hh"
namespace nix {
class StorePath;
@ -47,9 +48,9 @@ struct Generation
* distinct contents to avoid bloat, but nothing stops two
* non-adjacent generations from having the same contents.
*
* @todo Use `StorePath` instead of `Path`?
* @todo Use `StorePath` instead of `std::filesystem::path`?
*/
Path path;
std::filesystem::path path;
/**
* When the generation was created. This is extra metadata about the
@ -81,7 +82,7 @@ typedef std::list<Generation> Generations;
*
* Note that the current/active generation need not be the latest one.
*/
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(std::filesystem::path profile);
struct LocalFSStore;
@ -96,7 +97,7 @@ struct LocalFSStore;
* The behavior of reusing existing generations like this makes this
* procedure idempotent. It also avoids clutter.
*/
Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath);
std::filesystem::path createGeneration(LocalFSStore & store, std::filesystem::path profile, StorePath outPath);
/**
* Unconditionally delete a generation
@ -111,7 +112,7 @@ Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath);
*
* @todo Should we expose this at all?
*/
void deleteGeneration(const Path & profile, GenerationNumber gen);
void deleteGeneration(const std::filesystem::path & profile, GenerationNumber gen);
/**
* Delete the given set of generations.
@ -128,7 +129,8 @@ void deleteGeneration(const Path & profile, GenerationNumber gen);
* Trying to delete the currently active generation will fail, and cause
* no generations to be deleted.
*/
void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
void deleteGenerations(
const std::filesystem::path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
/**
* Delete generations older than `max` passed the current generation.
@ -142,7 +144,7 @@ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> &
* @param dryRun Log what would be deleted instead of actually doing
* so.
*/
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun);
void deleteGenerationsGreaterThan(const std::filesystem::path & profile, GenerationNumber max, bool dryRun);
/**
* Delete all generations other than the current one
@ -153,7 +155,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo
* @param dryRun Log what would be deleted instead of actually doing
* so.
*/
void deleteOldGenerations(const Path & profile, bool dryRun);
void deleteOldGenerations(const std::filesystem::path & profile, bool dryRun);
/**
* Delete generations older than `t`, except for the most recent one
@ -165,7 +167,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun);
* @param dryRun Log what would be deleted instead of actually doing
* so.
*/
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
void deleteGenerationsOlderThan(const std::filesystem::path & profile, time_t t, bool dryRun);
/**
* Parse a temp spec intended for `deleteGenerationsOlderThan()`.
@ -180,19 +182,19 @@ time_t parseOlderThanTimeSpec(std::string_view timeSpec);
*
* @todo Always use `switchGeneration()` instead, and delete this.
*/
void switchLink(Path link, Path target);
void switchLink(std::filesystem::path link, std::filesystem::path target);
/**
* Roll back a profile to the specified generation, or to the most
* recent one older than the current.
*/
void switchGeneration(const Path & profile, std::optional<GenerationNumber> dstGen, bool dryRun);
void switchGeneration(const std::filesystem::path & profile, std::optional<GenerationNumber> dstGen, bool dryRun);
/**
* Ensure exclusive access to a profile. Any command that modifies
* the profile first acquires this lock.
*/
void lockProfile(PathLocks & lock, const Path & profile);
void lockProfile(PathLocks & lock, const std::filesystem::path & profile);
/**
* Optimistic locking is used by long-running operations like `nix-env
@ -205,34 +207,34 @@ void lockProfile(PathLocks & lock, const Path & profile);
* store. Most of the time, only the user environment has to be
* rebuilt.
*/
std::string optimisticLockProfile(const Path & profile);
std::string optimisticLockProfile(const std::filesystem::path & profile);
/**
* Create and return the path to a directory suitable for storing the user’s
* profiles.
*/
Path profilesDir();
std::filesystem::path profilesDir();
/**
* Return the path to the profile directory for root (but don't try creating it)
*/
Path rootProfilesDir();
std::filesystem::path rootProfilesDir();
/**
* Create and return the path to the file used for storing the users's channels
*/
Path defaultChannelsDir();
std::filesystem::path defaultChannelsDir();
/**
* Return the path to the channel directory for root (but don't try creating it)
*/
Path rootChannelsDir();
std::filesystem::path rootChannelsDir();
/**
* Resolve the default profile (~/.nix-profile by default,
* $XDG_STATE_HOME/nix/profile if XDG Base Directory Support is enabled),
* and create if doesn't exist
*/
Path getDefaultProfile();
std::filesystem::path getDefaultProfile();
} // namespace nix

6
src/libstore/include/nix/store/store-api.hh
View file

@ -996,6 +996,12 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
*/
std::string showPaths(const PathSet & paths);
/**
* Display a set of paths in human-readable form (i.e., between quotes
* and separated by commas).
*/
std::string showPaths(const std::set<std::filesystem::path> paths);
std::optional<ValidPathInfo>
decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven = std::nullopt);

2
src/libstore/local-store.cc
View file

@ -1332,7 +1332,7 @@ std::pair<std::filesystem::path, AutoCloseFD> LocalStore::createTempDirInStore()
/* There is a slight possibility that `tmpDir' gets deleted by
the GC between createTempDir() and when we acquire a lock on it.
We'll repeat until 'tmpDir' exists and we've locked it. */
tmpDirFn = createTempDir(config->realStoreDir, "tmp");
tmpDirFn = createTempDir(std::filesystem::path{config->realStoreDir.get()}, "tmp");
tmpDirFd = openDirectory(tmpDirFn);
if (!tmpDirFd) {
continue;

2
src/libstore/pathlocks.cc
View file

@ -13,7 +13,7 @@ PathLocks::PathLocks()
{
}
PathLocks::PathLocks(const PathSet & paths, const std::string & waitMsg)
PathLocks::PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg)
: deletePaths(false)
{
lockPaths(paths, waitMsg);

87
src/libstore/profiles.cc
View file

@ -31,12 +31,12 @@ static std::optional<GenerationNumber> parseName(const std::string & profileName
return {};
}
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile)
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(std::filesystem::path profile)
{
Generations gens;
std::filesystem::path profileDir = dirOf(profile);
auto profileName = std::string(baseNameOf(profile));
std::filesystem::path profileDir = profile.parent_path();
auto profileName = profile.filename().string();
for (auto & i : DirectoryIterator{profileDir}) {
checkInterrupt();
@ -48,18 +48,20 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
gens.sort([](const Generation & a, const Generation & b) { return a.number < b.number; });
return {gens, pathExists(profile) ? parseName(profileName, readLink(profile)) : std::nullopt};
return {gens, pathExists(profile) ? parseName(profileName, readLink(profile).string()) : std::nullopt};
}
/**
* Create a generation name that can be parsed by `parseName()`.
*/
static Path makeName(const Path & profile, GenerationNumber num)
static std::filesystem::path makeName(const std::filesystem::path & profile, GenerationNumber num)
{
return fmt("%s-%s-link", profile, num);
/* NB std::filesystem::path when put in format strings is
quoted automatically. */
return fmt("%s-%s-link", profile.string(), num);
}
Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath)
std::filesystem::path createGeneration(LocalFSStore & store, std::filesystem::path profile, StorePath outPath)
{
/* The new generation number should be higher than old the
previous ones. */
@ -90,21 +92,24 @@ Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath)
to the permanent roots (of which the GC would have a stale
view). If we didn't do it this way, the GC might remove the
user environment etc. we've just built. */
Path generation = makeName(profile, num + 1);
store.addPermRoot(outPath, generation);
auto generation = makeName(profile, num + 1);
store.addPermRoot(outPath, generation.string());
return generation;
}
static void removeFile(const Path & path)
static void removeFile(const std::filesystem::path & path)
{
if (remove(path.c_str()) == -1)
throw SysError("cannot unlink '%1%'", path);
try {
std::filesystem::remove(path);
} catch (std::filesystem::filesystem_error & e) {
throw SysError("removing file '%1%'", path);
}
}
void deleteGeneration(const Path & profile, GenerationNumber gen)
void deleteGeneration(const std::filesystem::path & profile, GenerationNumber gen)
{
Path generation = makeName(profile, gen);
std::filesystem::path generation = makeName(profile, gen);
removeFile(generation);
}
@ -117,7 +122,7 @@ void deleteGeneration(const Path & profile, GenerationNumber gen)
*
* - We only actually delete if `dryRun` is false.
*/
static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
static void deleteGeneration2(const std::filesystem::path & profile, GenerationNumber gen, bool dryRun)
{
if (dryRun)
notice("would remove profile version %1%", gen);
@ -127,7 +132,8 @@ static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool d
}
}
void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun)
void deleteGenerations(
const std::filesystem::path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
@ -153,7 +159,7 @@ static inline void iterDropUntil(Generations & gens, auto && i, auto && cond)
;
}
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun)
void deleteGenerationsGreaterThan(const std::filesystem::path & profile, GenerationNumber max, bool dryRun)
{
if (max == 0)
throw Error("Must keep at least one generation, otherwise the current one would be deleted");
@ -178,7 +184,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo
deleteGeneration2(profile, i->number, dryRun);
}
void deleteOldGenerations(const Path & profile, bool dryRun)
void deleteOldGenerations(const std::filesystem::path & profile, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
@ -190,7 +196,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun)
deleteGeneration2(profile, i.number, dryRun);
}
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
void deleteGenerationsOlderThan(const std::filesystem::path & profile, time_t t, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
@ -238,16 +244,16 @@ time_t parseOlderThanTimeSpec(std::string_view timeSpec)
return curTime - *days * 24 * 3600;
}
void switchLink(Path link, Path target)
void switchLink(std::filesystem::path link, std::filesystem::path target)
{
/* Hacky. */
if (dirOf(target) == dirOf(link))
target = baseNameOf(target);
if (target.parent_path() == link.parent_path())
target = target.filename();
replaceSymlink(target, link);
}
void switchGeneration(const Path & profile, std::optional<GenerationNumber> dstGen, bool dryRun)
void switchGeneration(const std::filesystem::path & profile, std::optional<GenerationNumber> dstGen, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
@ -274,44 +280,47 @@ void switchGeneration(const Path & profile, std::optional<GenerationNumber> dstG
switchLink(profile, dst->path);
}
void lockProfile(PathLocks & lock, const Path & profile)
void lockProfile(PathLocks & lock, const std::filesystem::path & profile)
{
lock.lockPaths({profile}, fmt("waiting for lock on profile '%1%'", profile));
lock.setDeletion(true);
}
std::string optimisticLockProfile(const Path & profile)
std::string optimisticLockProfile(const std::filesystem::path & profile)
{
return pathExists(profile) ? readLink(profile) : "";
return pathExists(profile) ? readLink(profile).string() : "";
}
Path profilesDir()
std::filesystem::path profilesDir()
{
auto profileRoot = isRootUser() ? rootProfilesDir() : createNixStateDir() + "/profiles";
auto profileRoot = isRootUser() ? rootProfilesDir() : std::filesystem::path{createNixStateDir()} / "profiles";
createDirs(profileRoot);
return profileRoot;
}
Path rootProfilesDir()
std::filesystem::path rootProfilesDir()
{
return settings.nixStateDir + "/profiles/per-user/root";
return std::filesystem::path{settings.nixStateDir} / "profiles/per-user/root";
}
Path getDefaultProfile()
std::filesystem::path getDefaultProfile()
{
Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile";
std::filesystem::path profileLink = settings.useXDGBaseDirectories
? std::filesystem::path{createNixStateDir()} / "profile"
: std::filesystem::path{getHome()} / ".nix-profile";
try {
auto profile = profilesDir() + "/profile";
auto profile = profilesDir() / "profile";
if (!pathExists(profileLink)) {
replaceSymlink(profile, profileLink);
}
// Backwards compatibility measure: Make root's profile available as
// `.../default` as it's what NixOS and most of the init scripts expect
Path globalProfileLink = settings.nixStateDir + "/profiles/default";
auto globalProfileLink = std::filesystem::path{settings.nixStateDir} / "profiles" / "default";
if (isRootUser() && !pathExists(globalProfileLink)) {
replaceSymlink(profile, globalProfileLink);
}
return absPath(readLink(profileLink), dirOf(profileLink));
auto linkDir = profileLink.parent_path();
return absPath(readLink(profileLink), &linkDir);
} catch (Error &) {
return profileLink;
} catch (std::filesystem::filesystem_error &) {
@ -319,14 +328,14 @@ Path getDefaultProfile()
}
}
Path defaultChannelsDir()
std::filesystem::path defaultChannelsDir()
{
return profilesDir() + "/channels";
return profilesDir() / "channels";
}
Path rootChannelsDir()
std::filesystem::path rootChannelsDir()
{
return rootProfilesDir() + "/channels";
return rootProfilesDir() / "channels";
}
} // namespace nix

4
src/libstore/remote-store.cc
View file

@ -493,7 +493,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
auto conn(getConnection());
conn->to << WorkerProto::Op::RegisterDrvOutput;
if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) {
conn->to << info.id.to_string();
WorkerProto::write(*this, *conn, info.id);
conn->to << std::string(info.outPath.to_string());
} else {
WorkerProto::write(*this, *conn, info);
@ -513,7 +513,7 @@ void RemoteStore::queryRealisationUncached(
}
conn->to << WorkerProto::Op::QueryRealisation;
conn->to << id.to_string();
WorkerProto::write(*this, *conn, id);
conn.processStderr();
auto real = [&]() -> std::shared_ptr<const UnkeyedRealisation> {

5
src/libstore/store-api.cc
View file

@ -1126,6 +1126,11 @@ std::string StoreDirConfig::showPaths(const StorePathSet & paths) const
return s;
}
std::string showPaths(const std::set<std::filesystem::path> paths)
{
return concatStringsSep(", ", quoteFSPaths(paths));
}
std::string showPaths(const PathSet & paths)
{
return concatStringsSep(", ", quoteStrings(paths));

2
src/libstore/unix/build/darwin-derivation-builder.cc
View file

@ -174,7 +174,7 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl
/* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different
mechanisms to find temporary directories, so we want to open up a broader place for them to put their files,
if needed. */
Path globalTmpDir = canonPath(defaultTempDir(), true);
Path globalTmpDir = canonPath(defaultTempDir().string(), true);
/* They don't like trailing slashes on subpath directives */
while (!globalTmpDir.empty() && globalTmpDir.back() == '/')

22
src/libstore/unix/pathlocks.cc
View file

@ -13,18 +13,18 @@
namespace nix {
AutoCloseFD openLockFile(const Path & path, bool create)
AutoCloseFD openLockFile(const std::filesystem::path & path, bool create)
{
AutoCloseFD fd;
fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600);
if (!fd && (create || errno != ENOENT))
throw SysError("opening lock file '%1%'", path);
throw SysError("opening lock file %1%", path);
return fd;
}
void deleteLockFile(const Path & path, Descriptor desc)
void deleteLockFile(const std::filesystem::path & path, Descriptor desc)
{
/* Get rid of the lock file. Have to be careful not to introduce
races. Write a (meaningless) token to the file to indicate to
@ -69,7 +69,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait)
return true;
}
bool PathLocks::lockPaths(const PathSet & paths, const std::string & waitMsg, bool wait)
bool PathLocks::lockPaths(const std::set<std::filesystem::path> & paths, const std::string & waitMsg, bool wait)
{
assert(fds.empty());
@ -81,9 +81,9 @@ bool PathLocks::lockPaths(const PathSet & paths, const std::string & waitMsg, bo
preventing deadlocks. */
for (auto & path : paths) {
checkInterrupt();
Path lockPath = path + ".lock";
std::filesystem::path lockPath = path + ".lock";
debug("locking path '%1%'", path);
debug("locking path %1%", path);
AutoCloseFD fd;
@ -106,19 +106,19 @@ bool PathLocks::lockPaths(const PathSet & paths, const std::string & waitMsg, bo
}
}
debug("lock acquired on '%1%'", lockPath);
debug("lock acquired on %1%", lockPath);
/* Check that the lock file hasn't become stale (i.e.,
hasn't been unlinked). */
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError("statting lock file '%1%'", lockPath);
throw SysError("statting lock file %1%", lockPath);
if (st.st_size != 0)
/* This lock file has been unlinked, so we're holding
a lock on a deleted file. This means that other
processes may create and acquire a lock on
`lockPath', and proceed. So we must retry. */
debug("open lock file '%1%' has become stale", lockPath);
debug("open lock file %1% has become stale", lockPath);
else
break;
}
@ -137,9 +137,9 @@ void PathLocks::unlock()
deleteLockFile(i.second, i.first);
if (close(i.first) == -1)
printError("error (ignored): cannot close lock file on '%1%'", i.second);
printError("error (ignored): cannot close lock file on %1%", i.second);
debug("lock released on '%1%'", i.second);
debug("lock released on %1%", i.second);
}
fds.clear();

25
src/libstore/windows/pathlocks.cc
View file

@ -13,10 +13,10 @@ namespace nix {
using namespace nix::windows;
void deleteLockFile(const Path & path, Descriptor desc)
void deleteLockFile(const std::filesystem::path & path, Descriptor desc)
{
int exit = DeleteFileA(path.c_str());
int exit = DeleteFileW(path.c_str());
if (exit == 0)
warn("%s: &s", path, std::to_string(GetLastError()));
}
@ -28,17 +28,17 @@ void PathLocks::unlock()
deleteLockFile(i.second, i.first);
if (CloseHandle(i.first) == -1)
printError("error (ignored): cannot close lock file on '%1%'", i.second);
printError("error (ignored): cannot close lock file on %1%", i.second);
debug("lock released on '%1%'", i.second);
debug("lock released on %1%", i.second);
}
fds.clear();
}
AutoCloseFD openLockFile(const Path & path, bool create)
AutoCloseFD openLockFile(const std::filesystem::path & path, bool create)
{
AutoCloseFD desc = CreateFileA(
AutoCloseFD desc = CreateFileW(
path.c_str(),
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
@ -103,14 +103,15 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait)
}
}
bool PathLocks::lockPaths(const PathSet & paths, const std::string & waitMsg, bool wait)
bool PathLocks::lockPaths(const std::set<std::filesystem::path> & paths, const std::string & waitMsg, bool wait)
{
assert(fds.empty());
for (auto & path : paths) {
checkInterrupt();
Path lockPath = path + ".lock";
debug("locking path '%1%'", path);
std::filesystem::path lockPath = path;
lockPath += L".lock";
debug("locking path %1%", path);
AutoCloseFD fd;
@ -127,13 +128,13 @@ bool PathLocks::lockPaths(const PathSet & paths, const std::string & waitMsg, bo
}
}
debug("lock acquired on '%1%'", lockPath);
debug("lock acquired on %1%", lockPath);
struct _stat st;
if (_fstat(fromDescriptorReadOnly(fd.get()), &st) == -1)
throw SysError("statting lock file '%1%'", lockPath);
throw SysError("statting lock file %1%", lockPath);
if (st.st_size != 0)
debug("open lock file '%1%' has become stale", lockPath);
debug("open lock file %1% has become stale", lockPath);
else
break;
}

6
src/libutil-c/nix_api_util.cc
View file

@ -13,7 +13,11 @@ extern "C" {
nix_c_context * nix_c_context_create()
{
return new nix_c_context();
try {
return new nix_c_context();
} catch (...) {
return nullptr;
}
}
void nix_c_context_free(nix_c_context * context)

4
src/libutil-test-support/include/nix/util/tests/characterization.hh
View file

@ -31,16 +31,14 @@ static inline bool testAccept()
/**
* Mixin class for writing characterization tests
*/
class CharacterizationTest : public virtual ::testing::Test
struct CharacterizationTest : virtual ::testing::Test
{
protected:
/**
* While the "golden master" for this characterization test is
* located. It should not be shared with any other test.
*/
virtual std::filesystem::path goldenMaster(PathView testStem) const = 0;
public:
/**
* Golden test for reading
*

49
src/libutil-test-support/include/nix/util/tests/json-characterization.hh
View file

@ -5,6 +5,7 @@
#include <nlohmann/json.hpp>
#include "nix/util/types.hh"
#include "nix/util/ref.hh"
#include "nix/util/file-system.hh"
#include "nix/util/tests/characterization.hh"
@ -39,6 +40,49 @@ void writeJsonTest(CharacterizationTest & test, PathView testStem, const T & val
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });
}
/**
* Specialization for when we need to do "JSON -> `ref<T>`" in one
* direction, but "`const T &` -> JSON" in the other direction.
*
* We can't just return `const T &`, but it would be wasteful to
* requires a `const ref<T> &` double indirection (and mandatory shared
* pointer), so we break the symmetry as the best remaining option.
*/
template<typename T>
void writeJsonTest(CharacterizationTest & test, PathView testStem, const ref<T> & value)
{
using namespace nlohmann;
test.writeTest(
Path{testStem} + ".json",
[&]() -> json { return static_cast<json>(*value); },
[](const auto & file) { return json::parse(readFile(file)); },
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });
}
/**
* Golden test in the middle of something
*/
template<typename T>
void checkpointJson(CharacterizationTest & test, PathView testStem, const T & got)
{
using namespace nlohmann;
auto file = test.goldenMaster(Path{testStem} + ".json");
json gotJson = static_cast<json>(got);
if (testAccept()) {
std::filesystem::create_directories(file.parent_path());
writeFile(file, gotJson.dump(2) + "\n");
ADD_FAILURE() << "Updating golden master " << file;
} else {
json expectedJson = json::parse(readFile(file));
ASSERT_EQ(gotJson, expectedJson);
T expected = adl_serializer<T>::from_json(expectedJson);
ASSERT_EQ(got, expected);
}
}
/**
* Mixin class for writing characterization tests for `nlohmann::json`
* conversions for a given type.
@ -67,6 +111,11 @@ struct JsonCharacterizationTest : virtual CharacterizationTest
{
nix::writeJsonTest(*this, testStem, value);
}
void checkpointJson(PathView testStem, const T & value)
{
nix::checkpointJson(*this, testStem, value);
}
};
} // namespace nix

63
src/libutil-tests/file-system.cc
View file

@ -1,3 +1,4 @@
#include "nix/util/fs-sink.hh"
#include "nix/util/util.hh"
#include "nix/util/types.hh"
#include "nix/util/file-system.hh"
@ -318,4 +319,66 @@ TEST(DirectoryIterator, nonexistent)
ASSERT_THROW(DirectoryIterator("/schnitzel/darmstadt/pommes"), SysError);
}
/* ----------------------------------------------------------------------------
* openFileEnsureBeneathNoSymlinks
* --------------------------------------------------------------------------*/
#ifndef _WIN32
TEST(openFileEnsureBeneathNoSymlinks, works)
{
std::filesystem::path tmpDir = nix::createTempDir();
nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
using namespace nix::unix;
{
RestoreSink sink(/*startFsync=*/false);
sink.dstPath = tmpDir;
sink.dirFd = openDirectory(tmpDir);
sink.createDirectory(CanonPath("a"));
sink.createDirectory(CanonPath("c"));
sink.createDirectory(CanonPath("c/d"));
sink.createRegularFile(CanonPath("c/d/regular"), [](CreateRegularFileSink & crf) { crf("some contents"); });
sink.createSymlink(CanonPath("a/absolute_symlink"), tmpDir.string());
sink.createSymlink(CanonPath("a/relative_symlink"), "../.");
sink.createSymlink(CanonPath("a/broken_symlink"), "./nonexistent");
sink.createDirectory(CanonPath("a/b"), [](FileSystemObjectSink & dirSink, const CanonPath & relPath) {
dirSink.createDirectory(CanonPath("d"));
dirSink.createSymlink(CanonPath("c"), "./d");
});
sink.createDirectory(CanonPath("a/b/c/e")); // FIXME: This still follows symlinks
ASSERT_THROW(
sink.createDirectory(
CanonPath("a/b/c/f"), [](FileSystemObjectSink & dirSink, const CanonPath & relPath) {}),
SymlinkNotAllowed);
ASSERT_THROW(
sink.createRegularFile(
CanonPath("a/b/c/regular"), [](CreateRegularFileSink & crf) { crf("some contents"); }),
SymlinkNotAllowed);
}
AutoCloseFD dirFd = openDirectory(tmpDir);
auto open = [&](std::string_view path, int flags, mode_t mode = 0) {
return openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath(path), flags, mode);
};
EXPECT_THROW(open("a/absolute_symlink", O_RDONLY), SymlinkNotAllowed);
EXPECT_THROW(open("a/relative_symlink", O_RDONLY), SymlinkNotAllowed);
EXPECT_THROW(open("a/absolute_symlink/a", O_RDONLY), SymlinkNotAllowed);
EXPECT_THROW(open("a/absolute_symlink/c/d", O_RDONLY), SymlinkNotAllowed);
EXPECT_THROW(open("a/relative_symlink/c", O_RDONLY), SymlinkNotAllowed);
EXPECT_THROW(open("a/b/c/d", O_RDONLY), SymlinkNotAllowed);
EXPECT_EQ(open("a/broken_symlink", O_CREAT | O_WRONLY | O_EXCL, 0666), INVALID_DESCRIPTOR);
/* Sanity check, no symlink shenanigans and behaves the same as regular openat with O_EXCL | O_CREAT. */
EXPECT_EQ(errno, EEXIST);
EXPECT_THROW(open("a/absolute_symlink/broken_symlink", O_CREAT | O_WRONLY | O_EXCL, 0666), SymlinkNotAllowed);
EXPECT_EQ(open("c/d/regular/a", O_RDONLY), INVALID_DESCRIPTOR);
EXPECT_EQ(open("c/d/regular", O_RDONLY | O_DIRECTORY), INVALID_DESCRIPTOR);
EXPECT_TRUE(AutoCloseFD{open("c/d/regular", O_RDONLY)});
EXPECT_TRUE(AutoCloseFD{open("a/regular", O_CREAT | O_WRONLY | O_EXCL, 0666)});
}
#endif
} // namespace nix

4
src/libutil/args.cc
View file

@ -371,13 +371,13 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
d.completer(*completions, d.n, d.prefix);
}
Path Args::getCommandBaseDir() const
std::filesystem::path Args::getCommandBaseDir() const
{
assert(parent);
return parent->getCommandBaseDir();
}
Path RootArgs::getCommandBaseDir() const
std::filesystem::path RootArgs::getCommandBaseDir() const
{
return commandBaseDir;
}

21
src/libutil/current-process.cc
View file

@ -7,6 +7,7 @@
#include "nix/util/file-system.hh"
#include "nix/util/processes.hh"
#include "nix/util/signals.hh"
#include "nix/util/environment-variables.hh"
#include <math.h>
#ifdef __APPLE__
@ -65,13 +66,27 @@ void setStackSize(size_t stackSize)
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0 && static_cast<size_t>(limit.rlim_cur) < stackSize) {
savedStackSize = limit.rlim_cur;
limit.rlim_cur = std::min(static_cast<rlim_t>(stackSize), limit.rlim_max);
if (limit.rlim_max < static_cast<rlim_t>(stackSize)) {
if (getEnv("_NIX_TEST_NO_ENVIRONMENT_WARNINGS") != "1") {
logger->log(
lvlWarn,
HintFmt(
"Stack size hard limit is %1%, which is less than the desired %2%. If possible, increase the hard limit, e.g. with 'ulimit -Hs %3%'.",
limit.rlim_max,
stackSize,
stackSize / 1024)
.str());
}
}
auto requestedSize = std::min(static_cast<rlim_t>(stackSize), limit.rlim_max);
limit.rlim_cur = requestedSize;
if (setrlimit(RLIMIT_STACK, &limit) != 0) {
logger->log(
lvlError,
HintFmt(
"Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%",
"Failed to increase stack size from %1% to %2% (desired: %3%, maximum allowed: %4%): %5%",
savedStackSize,
requestedSize,
stackSize,
limit.rlim_max,
std::strerror(errno))
@ -109,7 +124,7 @@ std::optional<Path> getSelfExe()
{
static auto cached = []() -> std::optional<Path> {
#if defined(__linux__) || defined(__GNU__)
return readLink("/proc/self/exe");
return readLink(std::filesystem::path{"/proc/self/exe"});
#elif defined(__APPLE__)
char buf[1024];
uint32_t size = sizeof(buf);

25
src/libutil/file-system.cc
View file

@ -101,9 +101,11 @@ Path absPath(PathView path, std::optional<PathView> dir, bool resolveSymlinks)
return canonPath(path, resolveSymlinks);
}
std::filesystem::path absPath(const std::filesystem::path & path, bool resolveSymlinks)
std::filesystem::path
absPath(const std::filesystem::path & path, const std::filesystem::path * dir_, bool resolveSymlinks)
{
return absPath(path.string(), std::nullopt, resolveSymlinks);
std::optional<std::string> dir = dir_ ? std::optional<std::string>{dir_->string()} : std::nullopt;
return absPath(PathView{path.string()}, dir.transform([](auto & p) { return PathView(p); }), resolveSymlinks);
}
Path canonPath(PathView path, bool resolveSymlinks)
@ -242,10 +244,15 @@ bool pathAccessible(const std::filesystem::path & path)
}
}
Path readLink(const Path & path)
std::filesystem::path readLink(const std::filesystem::path & path)
{
checkInterrupt();
return std::filesystem::read_symlink(path).string();
return std::filesystem::read_symlink(path);
}
Path readLink(const Path & path)
{
return readLink(std::filesystem::path{path}).string();
}
std::string readFile(const Path & path)
@ -669,16 +676,16 @@ void AutoUnmount::cancel()
//////////////////////////////////////////////////////////////////////
std::string defaultTempDir()
std::filesystem::path defaultTempDir()
{
return getEnvNonEmpty("TMPDIR").value_or("/tmp");
}
Path createTempDir(const Path & tmpRoot, const Path & prefix, mode_t mode)
std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode)
{
while (1) {
checkInterrupt();
Path tmpDir = makeTempPath(tmpRoot, prefix);
std::filesystem::path tmpDir = makeTempPath(tmpRoot, prefix);
if (mkdir(
tmpDir.c_str()
#ifndef _WIN32 // TODO abstract mkdir perms for Windows
@ -720,11 +727,11 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
return {std::move(fd), tmpl};
}
Path makeTempPath(const Path & root, const Path & suffix)
std::filesystem::path makeTempPath(const std::filesystem::path & root, const std::string & suffix)
{
// start the counter at a random value to minimize issues with preexisting temp paths
static std::atomic<uint32_t> counter(std::random_device{}());
auto tmpRoot = canonPath(root.empty() ? defaultTempDir() : root, true);
auto tmpRoot = canonPath(root.empty() ? defaultTempDir().string() : root.string(), true);
return fmt("%1%/%2%-%3%-%4%", tmpRoot, suffix, getpid(), counter.fetch_add(1, std::memory_order_relaxed));
}

5
src/libutil/fs-sink.cc
View file

@ -84,7 +84,8 @@ void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallba
RestoreSink dirSink{startFsync};
dirSink.dstPath = append(dstPath, path);
dirSink.dirFd = ::openat(dirFd.get(), path.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
dirSink.dirFd =
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (!dirSink.dirFd)
throw SysError("opening directory '%s'", dirSink.dstPath.string());
@ -169,7 +170,7 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function<void(C
constexpr int flags = O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC;
if (!dirFd)
return ::open(p.c_str(), flags, 0666);
return ::openat(dirFd.get(), path.rel_c_str(), flags, 0666);
return unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, flags, 0666);
}();
#endif
;

2
src/libutil/include/nix/util/args.hh
View file

@ -59,7 +59,7 @@ public:
*
* This only returns the correct value after parseCmdline() has run.
*/
virtual Path getCommandBaseDir() const;
virtual std::filesystem::path getCommandBaseDir() const;
protected:

4
src/libutil/include/nix/util/args/root.hh
View file

@ -38,7 +38,7 @@ protected:
*
* @see getCommandBaseDir()
*/
Path commandBaseDir = ".";
std::filesystem::path commandBaseDir = ".";
public:
/** Parse the command line, throwing a UsageError if something goes
@ -48,7 +48,7 @@ public:
std::shared_ptr<Completions> completions;
Path getCommandBaseDir() const override;
std::filesystem::path getCommandBaseDir() const override;
protected:

60
src/libutil/include/nix/util/file-descriptor.hh
View file

@ -1,6 +1,7 @@
#pragma once
///@file
#include "nix/util/canon-path.hh"
#include "nix/util/types.hh"
#include "nix/util/error.hh"
@ -203,6 +204,26 @@ void closeOnExec(Descriptor fd);
} // namespace unix
#endif
#ifdef __linux__
namespace linux {
/**
* Wrapper around Linux's openat2 syscall introduced in Linux 5.6.
*
* @see https://man7.org/linux/man-pages/man2/openat2.2.html
* @see https://man7.org/linux/man-pages/man2/open_how.2type.html
v*
* @param flags O_* flags
* @param mode Mode for O_{CREAT,TMPFILE}
* @param resolve RESOLVE_* flags
*
* @return nullopt if openat2 is not supported by the kernel.
*/
std::optional<Descriptor> openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve);
} // namespace linux
#endif
#if defined(_WIN32) && _WIN32_WINNT >= 0x0600
namespace windows {
@ -212,6 +233,45 @@ std::wstring handleToFileName(Descriptor handle);
} // namespace windows
#endif
#ifndef _WIN32
namespace unix {
struct SymlinkNotAllowed : public Error
{
CanonPath path;
SymlinkNotAllowed(CanonPath path)
/* Can't provide better error message, since the parent directory is only known to the caller. */
: Error("relative path '%s' points to a symlink, which is not allowed", path.rel())
, path(std::move(path))
{
}
};
/**
* Safe(r) function to open \param path file relative to \param dirFd, while
* disallowing escaping from a directory and resolving any symlinks in the
* process.
*
* @note When not on Linux or when openat2 is not available this is implemented
* via openat single path component traversal. Uses RESOLVE_BENEATH with openat2
* or O_RESOLVE_BENEATH.
*
* @note Since this is Unix-only path is specified as CanonPath, which models
* Unix-style paths and ensures that there are no .. or . components.
*
* @param flags O_* flags
* @param mode Mode for O_{CREAT,TMPFILE}
*
* @pre path.isRoot() is false
*
* @throws SymlinkNotAllowed if any path components
*/
Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode = 0);
} // namespace unix
#endif
MakeError(EndOfFile, Error);
} // namespace nix

16
src/libutil/include/nix/util/file-system.hh
View file

@ -55,7 +55,8 @@ inline Path absPath(const Path & path, std::optional<PathView> dir = {}, bool re
return absPath(PathView{path}, dir, resolveSymlinks);
}
std::filesystem::path absPath(const std::filesystem::path & path, bool resolveSymlinks = false);
std::filesystem::path
absPath(const std::filesystem::path & path, const std::filesystem::path * dir = nullptr, bool resolveSymlinks = false);
/**
* Canonicalise a path by removing all `.` or `..` components and
@ -152,6 +153,12 @@ bool pathAccessible(const std::filesystem::path & path);
*/
Path readLink(const Path & path);
/**
* Read the contents (target) of a symbolic link. The result is not
* in any way canonicalised.
*/
std::filesystem::path readLink(const std::filesystem::path & path);
/**
* Open a `Descriptor` with read-only access to the given directory.
*/
@ -327,7 +334,8 @@ typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir;
/**
* Create a temporary directory.
*/
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", mode_t mode = 0755);
std::filesystem::path
createTempDir(const std::filesystem::path & tmpRoot = "", const std::string & prefix = "nix", mode_t mode = 0755);
/**
* Create a temporary file, returning a file handle and its path.
@ -337,7 +345,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
/**
* Return `TMPDIR`, or the default temporary directory if unset or empty.
*/
Path defaultTempDir();
std::filesystem::path defaultTempDir();
/**
* Interpret `exe` as a location in the ambient file system and return
@ -351,7 +359,7 @@ bool isExecutableFileAmbient(const std::filesystem::path & exe);
* The constructed path looks like `<root><suffix>-<pid>-<unique>`. To create a
* path nested in a directory, provide a suffix starting with `/`.
*/
Path makeTempPath(const Path & root, const Path & suffix = ".tmp");
std::filesystem::path makeTempPath(const std::filesystem::path & root, const std::string & suffix = ".tmp");
/**
* Used in various places.

24
src/libutil/include/nix/util/json-impls.hh
View file

@ -6,18 +6,30 @@
#include "nix/util/experimental-features.hh"
// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
#define JSON_IMPL_INNER_TO(TYPE) \
struct adl_serializer<TYPE> \
{ \
static void to_json(json & json, const TYPE & t); \
}
#define JSON_IMPL_INNER_FROM(TYPE) \
struct adl_serializer<TYPE> \
{ \
static TYPE from_json(const json & json); \
}
#define JSON_IMPL_INNER(TYPE) \
struct adl_serializer<TYPE> \
{ \
static TYPE from_json(const json & json); \
static void to_json(json & json, const TYPE & t); \
};
}
#define JSON_IMPL(TYPE) \
namespace nlohmann { \
using namespace nix; \
template<> \
JSON_IMPL_INNER(TYPE) \
#define JSON_IMPL(TYPE) \
namespace nlohmann { \
using namespace nix; \
template<> \
JSON_IMPL_INNER(TYPE); \
}
#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \

10
src/libutil/include/nix/util/memory-source-accessor.hh
View file

@ -188,23 +188,23 @@ using namespace nix;
#define ARG fso::Regular<RegularContents>
template<typename RegularContents>
JSON_IMPL_INNER(ARG)
JSON_IMPL_INNER(ARG);
#undef ARG
#define ARG fso::DirectoryT<Child>
template<typename Child>
JSON_IMPL_INNER(ARG)
JSON_IMPL_INNER(ARG);
#undef ARG
template<>
JSON_IMPL_INNER(fso::Symlink)
JSON_IMPL_INNER(fso::Symlink);
template<>
JSON_IMPL_INNER(fso::Opaque)
JSON_IMPL_INNER(fso::Opaque);
#define ARG fso::VariantT<RegularContents, recur>
template<typename RegularContents, bool recur>
JSON_IMPL_INNER(ARG)
JSON_IMPL_INNER(ARG);
#undef ARG
} // namespace nlohmann

19
src/libutil/include/nix/util/serialise.hh
View file

@ -447,18 +447,27 @@ struct LengthSource : Source
*/
struct LambdaSink : Sink
{
typedef std::function<void(std::string_view data)> lambda_t;
typedef std::function<void(std::string_view data)> data_t;
typedef std::function<void()> cleanup_t;
lambda_t lambda;
data_t dataFun;
cleanup_t cleanupFun;
LambdaSink(const lambda_t & lambda)
: lambda(lambda)
LambdaSink(
const data_t & dataFun, const cleanup_t & cleanupFun = []() {})
: dataFun(dataFun)
, cleanupFun(cleanupFun)
{
}
~LambdaSink()
{
cleanupFun();
}
void operator()(std::string_view data) override
{
lambda(data);
dataFun(data);
}
};

18
src/libutil/include/nix/util/users.hh
View file

@ -1,6 +1,8 @@
#pragma once
///@file
#include <filesystem>
#include "nix/util/types.hh"
#ifndef _WIN32
@ -15,43 +17,43 @@ std::string getUserName();
/**
* @return the given user's home directory from /etc/passwd.
*/
Path getHomeOf(uid_t userId);
std::filesystem::path getHomeOf(uid_t userId);
#endif
/**
* @return $HOME or the user's home directory from /etc/passwd.
*/
Path getHome();
std::filesystem::path getHome();
/**
* @return $NIX_CACHE_HOME or $XDG_CACHE_HOME/nix or $HOME/.cache/nix.
*/
Path getCacheDir();
std::filesystem::path getCacheDir();
/**
* @return $NIX_CONFIG_HOME or $XDG_CONFIG_HOME/nix or $HOME/.config/nix.
*/
Path getConfigDir();
std::filesystem::path getConfigDir();
/**
* @return the directories to search for user configuration files
*/
std::vector<Path> getConfigDirs();
std::vector<std::filesystem::path> getConfigDirs();
/**
* @return $NIX_DATA_HOME or $XDG_DATA_HOME/nix or $HOME/.local/share/nix.
*/
Path getDataDir();
std::filesystem::path getDataDir();
/**
* @return $NIX_STATE_HOME or $XDG_STATE_HOME/nix or $HOME/.local/state/nix.
*/
Path getStateDir();
std::filesystem::path getStateDir();
/**
* Create the Nix state directory and return the path to it.
*/
Path createNixStateDir();
std::filesystem::path createNixStateDir();
/**
* Perform tilde expansion on a path, replacing tilde with the user's

7
src/libutil/include/nix/util/util.hh
View file

@ -6,6 +6,7 @@
#include "nix/util/logging.hh"
#include "nix/util/strings.hh"
#include <filesystem>
#include <functional>
#include <map>
#include <sstream>
@ -58,6 +59,12 @@ Strings quoteStrings(const C & c, char quote = '\'')
return res;
}
inline Strings quoteFSPaths(const std::set<std::filesystem::path> & paths, char quote = '\'')
{
return paths | std::views::transform([&](const auto & p) { return quoteString(p.string(), quote); })
| std::ranges::to<Strings>();
}
/**
* Remove trailing whitespace from a string.
*

112
src/libutil/unix/file-descriptor.cc
View file

@ -1,3 +1,4 @@
#include "nix/util/canon-path.hh"
#include "nix/util/file-system.hh"
#include "nix/util/signals.hh"
#include "nix/util/finally.hh"
@ -7,6 +8,14 @@
#include <unistd.h>
#include <poll.h>
#if defined(__linux__) && defined(__NR_openat2)
# define HAVE_OPENAT2 1
# include <sys/syscall.h>
# include <linux/openat2.h>
#else
# define HAVE_OPENAT2 0
#endif
#include "util-config-private.hh"
#include "util-unix-config-private.hh"
@ -223,4 +232,107 @@ void unix::closeOnExec(int fd)
throw SysError("setting close-on-exec flag");
}
#ifdef __linux__
namespace linux {
std::optional<Descriptor> openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve)
{
# if HAVE_OPENAT2
/* Cache the result of whether openat2 is not supported. */
static std::atomic_flag unsupported{};
if (!unsupported.test()) {
/* No glibc wrapper yet, but there's a patch:
* https://patchwork.sourceware.org/project/glibc/patch/20251029200519.3203914-1-adhemerval.zanella@linaro.org/
*/
auto how = ::open_how{.flags = flags, .mode = mode, .resolve = resolve};
auto res = ::syscall(__NR_openat2, dirFd, path, &how, sizeof(how));
/* Cache that the syscall is not supported. */
if (res < 0 && errno == ENOSYS) {
unsupported.test_and_set();
return std::nullopt;
}
return res;
}
# endif
return std::nullopt;
}
} // namespace linux
#endif
static Descriptor
openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode)
{
AutoCloseFD parentFd;
auto nrComponents = std::ranges::distance(path);
assert(nrComponents >= 1);
auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */
auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; };
/* This rather convoluted loop is necessary to avoid TOCTOU when validating that
no inner path component is a symlink. */
for (auto it = components.begin(); it != components.end(); ++it) {
auto component = std::string(*it); /* Copy into a string to make NUL terminated. */
assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */
AutoCloseFD parentFd2 = ::openat(
getParentFd(), /* First iteration uses dirFd. */
component.c_str(),
O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC
#ifdef __linux__
| O_PATH /* Linux-specific optimization. Files are open only for path resolution purposes. */
#endif
#ifdef __FreeBSD__
| O_RESOLVE_BENEATH /* Further guard against any possible SNAFUs. */
#endif
);
if (!parentFd2) {
/* Construct the CanonPath for error message. */
auto path2 = std::ranges::fold_left(components.begin(), ++it, CanonPath::root, [](auto lhs, auto rhs) {
lhs.push(rhs);
return lhs;
});
if (errno == ENOTDIR) /* Path component might be a symlink. */ {
struct ::stat st;
if (::fstatat(getParentFd(), component.c_str(), &st, AT_SYMLINK_NOFOLLOW) == 0 && S_ISLNK(st.st_mode))
throw unix::SymlinkNotAllowed(path2);
errno = ENOTDIR; /* Restore the errno. */
} else if (errno == ELOOP) {
throw unix::SymlinkNotAllowed(path2);
}
return INVALID_DESCRIPTOR;
}
parentFd = std::move(parentFd2);
}
auto res = ::openat(getParentFd(), std::string(path.baseName().value()).c_str(), flags | O_NOFOLLOW, mode);
if (res < 0 && errno == ELOOP)
throw unix::SymlinkNotAllowed(path);
return res;
}
Descriptor unix::openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode)
{
assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */
assert(!path.isRoot());
#ifdef __linux__
auto maybeFd = linux::openat2(
dirFd, path.rel_c_str(), flags, static_cast<uint64_t>(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS);
if (maybeFd) {
if (*maybeFd < 0 && errno == ELOOP)
throw unix::SymlinkNotAllowed(path);
return *maybeFd;
}
#endif
return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode);
}
} // namespace nix

6
src/libutil/unix/users.cc
View file

@ -18,7 +18,7 @@ std::string getUserName()
return name;
}
Path getHomeOf(uid_t userId)
std::filesystem::path getHomeOf(uid_t userId)
{
std::vector<char> buf(16384);
struct passwd pwbuf;
@ -28,9 +28,9 @@ Path getHomeOf(uid_t userId)
return pw->pw_dir;
}
Path getHome()
std::filesystem::path getHome()
{
static Path homeDir = []() {
static std::filesystem::path homeDir = []() {
std::optional<std::string> unownedUserHomeDir = {};
auto homeDir = getEnv("HOME");
if (homeDir) {

41
src/libutil/users.cc
View file

@ -5,7 +5,7 @@
namespace nix {
Path getCacheDir()
std::filesystem::path getCacheDir()
{
auto dir = getEnv("NIX_CACHE_HOME");
if (dir) {
@ -13,14 +13,14 @@ Path getCacheDir()
} else {
auto xdgDir = getEnv("XDG_CACHE_HOME");
if (xdgDir) {
return *xdgDir + "/nix";
return std::filesystem::path{*xdgDir} / "nix";
} else {
return getHome() + "/.cache/nix";
return getHome() / ".cache" / "nix";
}
}
}
Path getConfigDir()
std::filesystem::path getConfigDir()
{
auto dir = getEnv("NIX_CONFIG_HOME");
if (dir) {
@ -28,26 +28,27 @@ Path getConfigDir()
} else {
auto xdgDir = getEnv("XDG_CONFIG_HOME");
if (xdgDir) {
return *xdgDir + "/nix";
return std::filesystem::path{*xdgDir} / "nix";
} else {
return getHome() + "/.config/nix";
return getHome() / ".config" / "nix";
}
}
}
std::vector<Path> getConfigDirs()
std::vector<std::filesystem::path> getConfigDirs()
{
Path configHome = getConfigDir();
std::filesystem::path configHome = getConfigDir();
auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg");
std::vector<Path> result = tokenizeString<std::vector<std::string>>(configDirs, ":");
for (auto & p : result) {
p += "/nix";
auto tokens = tokenizeString<std::vector<std::string>>(configDirs, ":");
std::vector<std::filesystem::path> result;
result.push_back(configHome);
for (auto & token : tokens) {
result.push_back(std::filesystem::path{token} / "nix");
}
result.insert(result.begin(), configHome);
return result;
}
Path getDataDir()
std::filesystem::path getDataDir()
{
auto dir = getEnv("NIX_DATA_HOME");
if (dir) {
@ -55,14 +56,14 @@ Path getDataDir()
} else {
auto xdgDir = getEnv("XDG_DATA_HOME");
if (xdgDir) {
return *xdgDir + "/nix";
return std::filesystem::path{*xdgDir} / "nix";
} else {
return getHome() + "/.local/share/nix";
return getHome() / ".local" / "share" / "nix";
}
}
}
Path getStateDir()
std::filesystem::path getStateDir()
{
auto dir = getEnv("NIX_STATE_HOME");
if (dir) {
@ -70,16 +71,16 @@ Path getStateDir()
} else {
auto xdgDir = getEnv("XDG_STATE_HOME");
if (xdgDir) {
return *xdgDir + "/nix";
return std::filesystem::path{*xdgDir} / "nix";
} else {
return getHome() + "/.local/state/nix";
return getHome() / ".local" / "state" / "nix";
}
}
}
Path createNixStateDir()
std::filesystem::path createNixStateDir()
{
Path dir = getStateDir();
std::filesystem::path dir = getStateDir();
createDirs(dir);
return dir;
}

6
src/libutil/windows/users.cc
View file

@ -35,10 +35,10 @@ std::string getUserName()
return name;
}
Path getHome()
std::filesystem::path getHome()
{
static Path homeDir = []() {
Path homeDir = getEnv("USERPROFILE").value_or("C:\\Users\\Default");
static std::filesystem::path homeDir = []() {
std::filesystem::path homeDir = getEnv("USERPROFILE").value_or("C:\\Users\\Default");
assert(!homeDir.empty());
return canonPath(homeDir);
}();

6
src/nix/app.cc
View file

@ -140,9 +140,9 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store)
auto res = unresolved;
auto builtContext = build(evalStore, store);
res.program = resolveString(*store, unresolved.program, builtContext);
if (!store->isInStore(res.program))
throw Error("app program '%s' is not in the Nix store", res.program);
res.program = resolveString(*store, unresolved.program.string(), builtContext);
if (!store->isInStore(res.program.string()))
throw Error("app program '%s' is not in the Nix store", res.program.string());
return res;
}

4
src/nix/derivation-add.cc
View file

@ -33,12 +33,10 @@ struct CmdAddDerivation : MixDryRun, StoreCommand
{
auto json = nlohmann::json::parse(drainFD(STDIN_FILENO));
auto drv = static_cast<Derivation>(json);
auto drv = Derivation::parseJsonAndValidate(*store, json);
auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun);
drv.checkInvariants(*store, drvPath);
writeDerivation(*store, drv, NoRepair, dryRun);
logger->cout("%s", store->printStorePath(drvPath));

2
src/nix/derivation-add.md
View file

@ -13,6 +13,6 @@ a Nix expression evaluates.
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
`nix derivation add` takes a single derivation in the JSON format.
See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format.
See [the manual](@docroot@/protocols/json/derivation/index.md) for a documentation of this format.
)""

2
src/nix/derivation-show.md
View file

@ -49,7 +49,7 @@ By default, this command only shows top-level derivations, but with
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
`nix derivation show` outputs a JSON map of [store path]s to derivations in JSON format.
See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format.
See [the manual](@docroot@/protocols/json/derivation/index.md) for a documentation of this format.
[store path]: @docroot@/store/store-path.md

Some files were not shown because too many files have changed in this diff Show more