mirror of
https://github.com/NixOS/nix.git
synced 2025-11-09 12:06:01 +01:00
Store StructuredAttrs directly in Derivation
Instead of parsing a structured attrs at some later point, we parsed it right away when parsing the A-Term format, and likewise serialize it to `__json = <JSON dump>` when serializing a derivation to A-Term. The JSON format can directly contain the JSON structured attrs without so encoding it, so we just do that.
This commit is contained in:
parent
b062730665
commit
8652b6b417
16 changed files with 177 additions and 109 deletions
|
|
@ -1363,7 +1363,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
|
||||||
|
|
||||||
/* Check whether attributes should be passed as a JSON file. */
|
/* Check whether attributes should be passed as a JSON file. */
|
||||||
using nlohmann::json;
|
using nlohmann::json;
|
||||||
std::optional<json> jsonObject;
|
std::optional<StructuredAttrs> jsonObject;
|
||||||
auto pos = v.determinePos(noPos);
|
auto pos = v.determinePos(noPos);
|
||||||
auto attr = attrs->find(state.sStructuredAttrs);
|
auto attr = attrs->find(state.sStructuredAttrs);
|
||||||
if (attr != attrs->end()
|
if (attr != attrs->end()
|
||||||
|
|
@ -1372,7 +1372,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
|
||||||
pos,
|
pos,
|
||||||
"while evaluating the `__structuredAttrs` "
|
"while evaluating the `__structuredAttrs` "
|
||||||
"attribute passed to builtins.derivationStrict"))
|
"attribute passed to builtins.derivationStrict"))
|
||||||
jsonObject = json::object();
|
jsonObject = StructuredAttrs{.structuredAttrs = json::object()};
|
||||||
|
|
||||||
/* Check whether null attributes should be ignored. */
|
/* Check whether null attributes should be ignored. */
|
||||||
bool ignoreNulls = false;
|
bool ignoreNulls = false;
|
||||||
|
|
@ -1484,7 +1484,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
|
||||||
if (i->name == state.sStructuredAttrs)
|
if (i->name == state.sStructuredAttrs)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
|
jsonObject->structuredAttrs.emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
|
||||||
|
|
||||||
if (i->name == state.sBuilder)
|
if (i->name == state.sBuilder)
|
||||||
drv.builder = state.forceString(*i->value, context, pos, context_below);
|
drv.builder = state.forceString(*i->value, context, pos, context_below);
|
||||||
|
|
@ -1532,23 +1532,26 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
|
auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
|
||||||
drv.env.emplace(key, s);
|
if (i->name == state.sJson) {
|
||||||
if (i->name == state.sBuilder)
|
|
||||||
drv.builder = std::move(s);
|
|
||||||
else if (i->name == state.sSystem)
|
|
||||||
drv.platform = std::move(s);
|
|
||||||
else if (i->name == state.sOutputHash)
|
|
||||||
outputHash = std::move(s);
|
|
||||||
else if (i->name == state.sOutputHashAlgo)
|
|
||||||
outputHashAlgo = parseHashAlgoOpt(s);
|
|
||||||
else if (i->name == state.sOutputHashMode)
|
|
||||||
handleHashMode(s);
|
|
||||||
else if (i->name == state.sOutputs)
|
|
||||||
handleOutputs(tokenizeString<Strings>(s));
|
|
||||||
else if (i->name == state.sJson)
|
|
||||||
warn(
|
warn(
|
||||||
"In derivation '%s': setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.",
|
"In derivation '%s': setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.",
|
||||||
drvName);
|
drvName);
|
||||||
|
drv.structuredAttrs = StructuredAttrs::parse(s);
|
||||||
|
} else {
|
||||||
|
drv.env.emplace(key, s);
|
||||||
|
if (i->name == state.sBuilder)
|
||||||
|
drv.builder = std::move(s);
|
||||||
|
else if (i->name == state.sSystem)
|
||||||
|
drv.platform = std::move(s);
|
||||||
|
else if (i->name == state.sOutputHash)
|
||||||
|
outputHash = std::move(s);
|
||||||
|
else if (i->name == state.sOutputHashAlgo)
|
||||||
|
outputHashAlgo = parseHashAlgoOpt(s);
|
||||||
|
else if (i->name == state.sOutputHashMode)
|
||||||
|
handleHashMode(s);
|
||||||
|
else if (i->name == state.sOutputs)
|
||||||
|
handleOutputs(tokenizeString<Strings>(s));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1560,8 +1563,10 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
|
||||||
}
|
}
|
||||||
|
|
||||||
if (jsonObject) {
|
if (jsonObject) {
|
||||||
drv.env.emplace("__json", jsonObject->dump());
|
/* The only other way `drv.structuredAttrs` can be set is when
|
||||||
jsonObject.reset();
|
`jsonObject` is not set. */
|
||||||
|
assert(!drv.structuredAttrs);
|
||||||
|
drv.structuredAttrs = std::move(*jsonObject);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Everything in the context of the strings in the derivation
|
/* Everything in the context of the strings in the derivation
|
||||||
|
|
|
||||||
|
|
@ -108,10 +108,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_TRUE(!parsedDrv);
|
EXPECT_TRUE(!got.structuredAttrs);
|
||||||
|
|
||||||
EXPECT_EQ(options.additionalSandboxProfile, "");
|
EXPECT_EQ(options.additionalSandboxProfile, "");
|
||||||
EXPECT_EQ(options.noChroot, false);
|
EXPECT_EQ(options.noChroot, false);
|
||||||
|
|
@ -143,8 +142,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
|
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
|
||||||
});
|
});
|
||||||
|
|
@ -157,8 +155,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
|
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
|
||||||
});
|
});
|
||||||
|
|
@ -171,10 +168,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_TRUE(!parsedDrv);
|
EXPECT_TRUE(!got.structuredAttrs);
|
||||||
|
|
||||||
EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
|
EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
|
||||||
EXPECT_EQ(options.noChroot, true);
|
EXPECT_EQ(options.noChroot, true);
|
||||||
|
|
@ -195,8 +191,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(
|
EXPECT_EQ(
|
||||||
options.exportReferencesGraph,
|
options.exportReferencesGraph,
|
||||||
|
|
@ -245,8 +240,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(
|
EXPECT_EQ(
|
||||||
options.exportReferencesGraph,
|
options.exportReferencesGraph,
|
||||||
|
|
@ -298,10 +292,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_TRUE(parsedDrv);
|
EXPECT_TRUE(got.structuredAttrs);
|
||||||
|
|
||||||
EXPECT_EQ(options.additionalSandboxProfile, "");
|
EXPECT_EQ(options.additionalSandboxProfile, "");
|
||||||
EXPECT_EQ(options.noChroot, false);
|
EXPECT_EQ(options.noChroot, false);
|
||||||
|
|
@ -332,8 +325,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
|
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
|
||||||
});
|
});
|
||||||
|
|
@ -346,8 +338,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
|
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
|
||||||
});
|
});
|
||||||
|
|
@ -360,10 +351,9 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_TRUE(parsedDrv);
|
EXPECT_TRUE(got.structuredAttrs);
|
||||||
|
|
||||||
EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
|
EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
|
||||||
EXPECT_EQ(options.noChroot, true);
|
EXPECT_EQ(options.noChroot, true);
|
||||||
|
|
@ -394,8 +384,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(
|
EXPECT_EQ(
|
||||||
options.exportReferencesGraph,
|
options.exportReferencesGraph,
|
||||||
|
|
@ -448,8 +437,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
|
||||||
|
|
||||||
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(got.env);
|
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
|
||||||
DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
|
|
||||||
EXPECT_EQ(
|
EXPECT_EQ(
|
||||||
options.exportReferencesGraph,
|
options.exportReferencesGraph,
|
||||||
|
|
|
||||||
|
|
@ -222,7 +222,7 @@ Derivation makeSimpleDrv(const Store & store)
|
||||||
"bar",
|
"bar",
|
||||||
"baz",
|
"baz",
|
||||||
};
|
};
|
||||||
drv.env = {
|
drv.env = StringPairs{
|
||||||
{
|
{
|
||||||
"BIG_BAD",
|
"BIG_BAD",
|
||||||
"WOLF",
|
"WOLF",
|
||||||
|
|
@ -284,7 +284,7 @@ Derivation makeDynDepDerivation(const Store & store)
|
||||||
"bar",
|
"bar",
|
||||||
"baz",
|
"baz",
|
||||||
};
|
};
|
||||||
drv.env = {
|
drv.env = StringPairs{
|
||||||
{
|
{
|
||||||
"BIG_BAD",
|
"BIG_BAD",
|
||||||
"WOLF",
|
"WOLF",
|
||||||
|
|
|
||||||
|
|
@ -32,12 +32,9 @@ DerivationBuildingGoal::DerivationBuildingGoal(
|
||||||
{
|
{
|
||||||
drv = std::make_unique<Derivation>(drv_);
|
drv = std::make_unique<Derivation>(drv_);
|
||||||
|
|
||||||
if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) {
|
|
||||||
parsedDrv = std::make_unique<StructuredAttrs>(*parsedOpt);
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
drvOptions =
|
drvOptions =
|
||||||
std::make_unique<DerivationOptions>(DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get()));
|
std::make_unique<DerivationOptions>(DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs));
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
|
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
|
||||||
throw;
|
throw;
|
||||||
|
|
@ -661,7 +658,6 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||||
buildMode,
|
buildMode,
|
||||||
buildResult,
|
buildResult,
|
||||||
*drv,
|
*drv,
|
||||||
parsedDrv.get(),
|
|
||||||
*drvOptions,
|
*drvOptions,
|
||||||
inputPaths,
|
inputPaths,
|
||||||
initialOutputs,
|
initialOutputs,
|
||||||
|
|
|
||||||
|
|
@ -63,9 +63,8 @@ Goal::Co DerivationGoal::haveDerivation()
|
||||||
trace("have derivation");
|
trace("have derivation");
|
||||||
|
|
||||||
auto drvOptions = [&]() -> DerivationOptions {
|
auto drvOptions = [&]() -> DerivationOptions {
|
||||||
auto parsedOpt = StructuredAttrs::tryParse(drv->env);
|
|
||||||
try {
|
try {
|
||||||
return DerivationOptions::fromStructuredAttrs(drv->env, parsedOpt ? &*parsedOpt : nullptr);
|
return DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
|
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
|
||||||
throw;
|
throw;
|
||||||
|
|
|
||||||
|
|
@ -92,6 +92,12 @@ using OutputChecks = DerivationOptions::OutputChecks;
|
||||||
|
|
||||||
using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
|
using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
|
||||||
|
|
||||||
|
DerivationOptions DerivationOptions::fromStructuredAttrs(
|
||||||
|
const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn)
|
||||||
|
{
|
||||||
|
return fromStructuredAttrs(env, parsed ? &*parsed : nullptr);
|
||||||
|
}
|
||||||
|
|
||||||
DerivationOptions
|
DerivationOptions
|
||||||
DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
|
DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -452,6 +452,7 @@ Derivation parseDerivation(
|
||||||
expect(str, ")");
|
expect(str, ")");
|
||||||
drv.env.insert_or_assign(std::move(name), std::move(value));
|
drv.env.insert_or_assign(std::move(name), std::move(value));
|
||||||
}
|
}
|
||||||
|
drv.structuredAttrs = StructuredAttrs::tryExtract(drv.env);
|
||||||
|
|
||||||
expect(str, ")");
|
expect(str, ")");
|
||||||
return drv;
|
return drv;
|
||||||
|
|
@ -685,16 +686,28 @@ std::string Derivation::unparse(
|
||||||
|
|
||||||
s += ",[";
|
s += ",[";
|
||||||
first = true;
|
first = true;
|
||||||
for (auto & i : env) {
|
|
||||||
if (first)
|
auto unparseEnv = [&](const StringPairs atermEnv) {
|
||||||
first = false;
|
for (auto & i : atermEnv) {
|
||||||
else
|
if (first)
|
||||||
|
first = false;
|
||||||
|
else
|
||||||
|
s += ',';
|
||||||
|
s += '(';
|
||||||
|
printString(s, i.first);
|
||||||
s += ',';
|
s += ',';
|
||||||
s += '(';
|
printString(s, maskOutputs && outputs.count(i.first) ? "" : i.second);
|
||||||
printString(s, i.first);
|
s += ')';
|
||||||
s += ',';
|
}
|
||||||
printString(s, maskOutputs && outputs.count(i.first) ? "" : i.second);
|
};
|
||||||
s += ')';
|
|
||||||
|
StructuredAttrs::checkKeyNotInUse(env);
|
||||||
|
if (structuredAttrs) {
|
||||||
|
StringPairs scratch = env;
|
||||||
|
scratch.insert(structuredAttrs->unparse());
|
||||||
|
unparseEnv(scratch);
|
||||||
|
} else {
|
||||||
|
unparseEnv(env);
|
||||||
}
|
}
|
||||||
|
|
||||||
s += "])";
|
s += "])";
|
||||||
|
|
@ -948,6 +961,7 @@ Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivati
|
||||||
auto value = readString(in);
|
auto value = readString(in);
|
||||||
drv.env[key] = value;
|
drv.env[key] = value;
|
||||||
}
|
}
|
||||||
|
drv.structuredAttrs = StructuredAttrs::tryExtract(drv.env);
|
||||||
|
|
||||||
return in;
|
return in;
|
||||||
}
|
}
|
||||||
|
|
@ -983,9 +997,21 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
|
||||||
}
|
}
|
||||||
CommonProto::write(store, CommonProto::WriteConn{.to = out}, drv.inputSrcs);
|
CommonProto::write(store, CommonProto::WriteConn{.to = out}, drv.inputSrcs);
|
||||||
out << drv.platform << drv.builder << drv.args;
|
out << drv.platform << drv.builder << drv.args;
|
||||||
out << drv.env.size();
|
|
||||||
for (auto & i : drv.env)
|
auto writeEnv = [&](const StringPairs atermEnv) {
|
||||||
out << i.first << i.second;
|
out << atermEnv.size();
|
||||||
|
for (auto & [k, v] : atermEnv)
|
||||||
|
out << k << v;
|
||||||
|
};
|
||||||
|
|
||||||
|
StructuredAttrs::checkKeyNotInUse(drv.env);
|
||||||
|
if (drv.structuredAttrs) {
|
||||||
|
StringPairs scratch = drv.env;
|
||||||
|
scratch.insert(drv.structuredAttrs->unparse());
|
||||||
|
writeEnv(scratch);
|
||||||
|
} else {
|
||||||
|
writeEnv(drv.env);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string hashPlaceholder(const OutputNameView outputName)
|
std::string hashPlaceholder(const OutputNameView outputName)
|
||||||
|
|
@ -1017,6 +1043,17 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites)
|
||||||
newEnv.emplace(envName, envValue);
|
newEnv.emplace(envName, envValue);
|
||||||
}
|
}
|
||||||
env = std::move(newEnv);
|
env = std::move(newEnv);
|
||||||
|
|
||||||
|
if (structuredAttrs) {
|
||||||
|
// TODO rewrite the JSON AST properly, rather than dump parse round trip.
|
||||||
|
auto [k, jsonS] = structuredAttrs->unparse();
|
||||||
|
jsonS = rewriteStrings(jsonS, rewrites);
|
||||||
|
StringPairs newEnv;
|
||||||
|
newEnv.insert(std::pair{k, std::move(jsonS)});
|
||||||
|
auto newStructuredAttrs = StructuredAttrs::tryExtract(newEnv);
|
||||||
|
assert(newStructuredAttrs);
|
||||||
|
structuredAttrs = std::move(*newStructuredAttrs);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
||||||
|
|
@ -1338,10 +1375,8 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
|
||||||
res["args"] = args;
|
res["args"] = args;
|
||||||
res["env"] = env;
|
res["env"] = env;
|
||||||
|
|
||||||
if (auto it = env.find("__json"); it != env.end()) {
|
if (structuredAttrs)
|
||||||
res["env"].erase("__json");
|
res["structuredAttrs"] = structuredAttrs->structuredAttrs;
|
||||||
res["structuredAttrs"] = nlohmann::json::parse(it->second);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
@ -1411,7 +1446,7 @@ Derivation Derivation::fromJSON(
|
||||||
}
|
}
|
||||||
|
|
||||||
if (auto structuredAttrs = get(json, "structuredAttrs"))
|
if (auto structuredAttrs = get(json, "structuredAttrs"))
|
||||||
res.env.insert_or_assign("__json", structuredAttrs->dump());
|
res.structuredAttrs = StructuredAttrs{*structuredAttrs};
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
///@file
|
///@file
|
||||||
|
|
||||||
#include "nix/store/parsed-derivations.hh"
|
|
||||||
#include "nix/store/derivations.hh"
|
#include "nix/store/derivations.hh"
|
||||||
|
#include "nix/store/parsed-derivations.hh"
|
||||||
#include "nix/store/derivation-options.hh"
|
#include "nix/store/derivation-options.hh"
|
||||||
#include "nix/store/build/derivation-building-misc.hh"
|
#include "nix/store/build/derivation-building-misc.hh"
|
||||||
#include "nix/store/outputs-spec.hh"
|
#include "nix/store/outputs-spec.hh"
|
||||||
|
|
@ -39,7 +39,6 @@ struct DerivationBuildingGoal : public Goal
|
||||||
*/
|
*/
|
||||||
std::unique_ptr<Derivation> drv;
|
std::unique_ptr<Derivation> drv;
|
||||||
|
|
||||||
std::unique_ptr<StructuredAttrs> parsedDrv;
|
|
||||||
std::unique_ptr<DerivationOptions> drvOptions;
|
std::unique_ptr<DerivationOptions> drvOptions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -176,6 +176,9 @@ struct DerivationOptions
|
||||||
static DerivationOptions
|
static DerivationOptions
|
||||||
fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true);
|
fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true);
|
||||||
|
|
||||||
|
static DerivationOptions
|
||||||
|
fromStructuredAttrs(const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn = true);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param drv Must be the same derivation we parsed this from. In
|
* @param drv Must be the same derivation we parsed this from. In
|
||||||
* the future we'll flip things around so a `BasicDerivation` has
|
* the future we'll flip things around so a `BasicDerivation` has
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@
|
||||||
#include "nix/store/content-address.hh"
|
#include "nix/store/content-address.hh"
|
||||||
#include "nix/util/repair-flag.hh"
|
#include "nix/util/repair-flag.hh"
|
||||||
#include "nix/store/derived-path-map.hh"
|
#include "nix/store/derived-path-map.hh"
|
||||||
|
#include "nix/store/parsed-derivations.hh"
|
||||||
#include "nix/util/sync.hh"
|
#include "nix/util/sync.hh"
|
||||||
#include "nix/util/variant-wrapper.hh"
|
#include "nix/util/variant-wrapper.hh"
|
||||||
|
|
||||||
|
|
@ -286,7 +287,12 @@ struct BasicDerivation
|
||||||
std::string platform;
|
std::string platform;
|
||||||
Path builder;
|
Path builder;
|
||||||
Strings args;
|
Strings args;
|
||||||
|
/**
|
||||||
|
* Must not contain the key `__json`, at least in order to serialize to A-Term.
|
||||||
|
*/
|
||||||
StringPairs env;
|
StringPairs env;
|
||||||
|
std::optional<StructuredAttrs> structuredAttrs;
|
||||||
|
|
||||||
std::string name;
|
std::string name;
|
||||||
|
|
||||||
BasicDerivation() = default;
|
BasicDerivation() = default;
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,30 @@ struct StructuredAttrs
|
||||||
{
|
{
|
||||||
nlohmann::json structuredAttrs;
|
nlohmann::json structuredAttrs;
|
||||||
|
|
||||||
static std::optional<StructuredAttrs> tryParse(const StringPairs & env);
|
bool operator==(const StructuredAttrs &) const = default;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unconditionally parse from a JSON string. Used by `tryExtract`.
|
||||||
|
*/
|
||||||
|
static StructuredAttrs parse(const std::string & encoded);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Like `tryParse`, but removes the env var which encoded the structured
|
||||||
|
* attrs from the map if one is found.
|
||||||
|
*/
|
||||||
|
static std::optional<StructuredAttrs> tryExtract(StringPairs & env);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Opposite of `tryParse`, at least if one makes a map from this
|
||||||
|
* single key-value PR.
|
||||||
|
*/
|
||||||
|
std::pair<std::string_view, std::string> unparse() const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensures that the structured attrs "env var" is not in used, so we
|
||||||
|
* are free to use it instead.
|
||||||
|
*/
|
||||||
|
static void checkKeyNotInUse(const StringPairs & env);
|
||||||
|
|
||||||
nlohmann::json prepareStructuredAttrs(
|
nlohmann::json prepareStructuredAttrs(
|
||||||
Store & store,
|
Store & store,
|
||||||
|
|
|
||||||
|
|
@ -224,13 +224,11 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
|
auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(drv->env);
|
|
||||||
DerivationOptions drvOptions;
|
DerivationOptions drvOptions;
|
||||||
try {
|
try {
|
||||||
// FIXME: this is a lot of work just to get the value
|
// FIXME: this is a lot of work just to get the value
|
||||||
// of `allowSubstitutes`.
|
// of `allowSubstitutes`.
|
||||||
drvOptions =
|
drvOptions = DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
|
||||||
DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv ? &*parsedDrv : nullptr);
|
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath));
|
e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath));
|
||||||
throw;
|
throw;
|
||||||
|
|
|
||||||
|
|
@ -8,20 +8,41 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
std::optional<StructuredAttrs> StructuredAttrs::tryParse(const StringPairs & env)
|
static constexpr std::string_view envVarName = "__json";
|
||||||
|
|
||||||
|
StructuredAttrs StructuredAttrs::parse(const std::string & encoded)
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
return StructuredAttrs{
|
||||||
|
.structuredAttrs = nlohmann::json::parse(encoded),
|
||||||
|
};
|
||||||
|
} catch (std::exception & e) {
|
||||||
|
throw Error("cannot process __json attribute: %s", e.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::optional<StructuredAttrs> StructuredAttrs::tryExtract(StringPairs & env)
|
||||||
{
|
{
|
||||||
/* Parse the __json attribute, if any. */
|
/* Parse the __json attribute, if any. */
|
||||||
auto jsonAttr = env.find("__json");
|
auto jsonAttr = env.find(envVarName);
|
||||||
if (jsonAttr != env.end()) {
|
if (jsonAttr != env.end()) {
|
||||||
try {
|
auto encoded = std::move(jsonAttr->second);
|
||||||
return StructuredAttrs{
|
env.erase(jsonAttr);
|
||||||
.structuredAttrs = nlohmann::json::parse(jsonAttr->second),
|
return parse(encoded);
|
||||||
};
|
} else
|
||||||
} catch (std::exception & e) {
|
return {};
|
||||||
throw Error("cannot process __json attribute: %s", e.what());
|
}
|
||||||
}
|
|
||||||
}
|
std::pair<std::string_view, std::string> StructuredAttrs::unparse() const
|
||||||
return {};
|
{
|
||||||
|
return {envVarName, structuredAttrs.dump()};
|
||||||
|
}
|
||||||
|
|
||||||
|
void StructuredAttrs::checkKeyNotInUse(const StringPairs & env)
|
||||||
|
{
|
||||||
|
if (env.count(envVarName))
|
||||||
|
throw Error(
|
||||||
|
"Cannot have an environment variable named '__json'. This key is reserved for encoding structured attrs");
|
||||||
}
|
}
|
||||||
|
|
||||||
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
|
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
|
||||||
|
|
@ -175,4 +196,5 @@ std::string StructuredAttrs::writeShell(const nlohmann::json & json)
|
||||||
|
|
||||||
return jsonSh;
|
return jsonSh;
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
|
||||||
|
|
@ -809,7 +809,7 @@ void DerivationBuilderImpl::startBuilder()
|
||||||
writeStructuredAttrs();
|
writeStructuredAttrs();
|
||||||
|
|
||||||
/* Handle exportReferencesGraph(), if set. */
|
/* Handle exportReferencesGraph(), if set. */
|
||||||
if (!parsedDrv) {
|
if (!drv.structuredAttrs) {
|
||||||
for (auto & [fileName, ss] : drvOptions.exportReferencesGraph) {
|
for (auto & [fileName, ss] : drvOptions.exportReferencesGraph) {
|
||||||
StorePathSet storePathSet;
|
StorePathSet storePathSet;
|
||||||
for (auto & storePathS : ss) {
|
for (auto & storePathS : ss) {
|
||||||
|
|
@ -1081,7 +1081,7 @@ void DerivationBuilderImpl::initEnv()
|
||||||
/* In non-structured mode, set all bindings either directory in the
|
/* In non-structured mode, set all bindings either directory in the
|
||||||
environment or via a file, as specified by
|
environment or via a file, as specified by
|
||||||
`DerivationOptions::passAsFile`. */
|
`DerivationOptions::passAsFile`. */
|
||||||
if (!parsedDrv) {
|
if (!drv.structuredAttrs) {
|
||||||
for (auto & i : drv.env) {
|
for (auto & i : drv.env) {
|
||||||
if (drvOptions.passAsFile.find(i.first) == drvOptions.passAsFile.end()) {
|
if (drvOptions.passAsFile.find(i.first) == drvOptions.passAsFile.end()) {
|
||||||
env[i.first] = i.second;
|
env[i.first] = i.second;
|
||||||
|
|
@ -1149,8 +1149,8 @@ void DerivationBuilderImpl::initEnv()
|
||||||
|
|
||||||
void DerivationBuilderImpl::writeStructuredAttrs()
|
void DerivationBuilderImpl::writeStructuredAttrs()
|
||||||
{
|
{
|
||||||
if (parsedDrv) {
|
if (drv.structuredAttrs) {
|
||||||
auto json = parsedDrv->prepareStructuredAttrs(store, drvOptions, inputPaths, drv.outputs);
|
auto json = drv.structuredAttrs->prepareStructuredAttrs(store, drvOptions, inputPaths, drv.outputs);
|
||||||
nlohmann::json rewritten;
|
nlohmann::json rewritten;
|
||||||
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
|
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
|
||||||
/* The placeholder must have a rewrite, so we use it to cover both the
|
/* The placeholder must have a rewrite, so we use it to cover both the
|
||||||
|
|
|
||||||
|
|
@ -27,15 +27,6 @@ struct DerivationBuilderParams
|
||||||
*/
|
*/
|
||||||
const Derivation & drv;
|
const Derivation & drv;
|
||||||
|
|
||||||
/**
|
|
||||||
* The "structured attrs" of `drv`, if it has them.
|
|
||||||
*
|
|
||||||
* @todo this should be part of `Derivation`.
|
|
||||||
*
|
|
||||||
* @todo this should be renamed from `parsedDrv`.
|
|
||||||
*/
|
|
||||||
const StructuredAttrs * parsedDrv;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The derivation options of `drv`.
|
* The derivation options of `drv`.
|
||||||
*
|
*
|
||||||
|
|
@ -63,14 +54,12 @@ struct DerivationBuilderParams
|
||||||
const BuildMode & buildMode,
|
const BuildMode & buildMode,
|
||||||
BuildResult & buildResult,
|
BuildResult & buildResult,
|
||||||
const Derivation & drv,
|
const Derivation & drv,
|
||||||
const StructuredAttrs * parsedDrv,
|
|
||||||
const DerivationOptions & drvOptions,
|
const DerivationOptions & drvOptions,
|
||||||
const StorePathSet & inputPaths,
|
const StorePathSet & inputPaths,
|
||||||
std::map<std::string, InitialOutput> & initialOutputs)
|
std::map<std::string, InitialOutput> & initialOutputs)
|
||||||
: drvPath{drvPath}
|
: drvPath{drvPath}
|
||||||
, buildResult{buildResult}
|
, buildResult{buildResult}
|
||||||
, drv{drv}
|
, drv{drv}
|
||||||
, parsedDrv{parsedDrv}
|
|
||||||
, drvOptions{drvOptions}
|
, drvOptions{drvOptions}
|
||||||
, inputPaths{inputPaths}
|
, inputPaths{inputPaths}
|
||||||
, initialOutputs{initialOutputs}
|
, initialOutputs{initialOutputs}
|
||||||
|
|
|
||||||
|
|
@ -555,10 +555,9 @@ static void main_nix_build(int argc, char ** argv)
|
||||||
env["NIX_STORE"] = store->storeDir;
|
env["NIX_STORE"] = store->storeDir;
|
||||||
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
|
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
|
||||||
|
|
||||||
auto parsedDrv = StructuredAttrs::tryParse(drv.env);
|
|
||||||
DerivationOptions drvOptions;
|
DerivationOptions drvOptions;
|
||||||
try {
|
try {
|
||||||
drvOptions = DerivationOptions::fromStructuredAttrs(drv.env, parsedDrv ? &*parsedDrv : nullptr);
|
drvOptions = DerivationOptions::fromStructuredAttrs(drv.env, drv.structuredAttrs);
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath()));
|
e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath()));
|
||||||
throw;
|
throw;
|
||||||
|
|
@ -577,7 +576,7 @@ static void main_nix_build(int argc, char ** argv)
|
||||||
|
|
||||||
std::string structuredAttrsRC;
|
std::string structuredAttrsRC;
|
||||||
|
|
||||||
if (parsedDrv) {
|
if (drv.structuredAttrs) {
|
||||||
StorePathSet inputs;
|
StorePathSet inputs;
|
||||||
|
|
||||||
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
|
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
|
||||||
|
|
@ -596,7 +595,7 @@ static void main_nix_build(int argc, char ** argv)
|
||||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
|
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
|
||||||
accumInputClosure(inputDrv, inputNode);
|
accumInputClosure(inputDrv, inputNode);
|
||||||
|
|
||||||
auto json = parsedDrv->prepareStructuredAttrs(*store, drvOptions, inputs, drv.outputs);
|
auto json = drv.structuredAttrs->prepareStructuredAttrs(*store, drvOptions, inputs, drv.outputs);
|
||||||
|
|
||||||
structuredAttrsRC = StructuredAttrs::writeShell(json);
|
structuredAttrsRC = StructuredAttrs::writeShell(json);
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue