nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-09 12:06:01 +01:00
Code Activity

Store DerivationOption instead Derivation, put in JSON format

Browse source 
This opens the door to derivations that *directly* specify their
options, rather than "stealing" environment variables to do so.

The A-Term derivation format used on disk didn't change --- we cannot do
that for existing derivations, so those derivations will continue to not
support separate options. But having a more flexible JSON format opens
the door to extending the on-disk format in others, like directly using
JSON, using CBOR, etc.

Co-authored-by: HaeNoe <git@haenoe.party>
Co-authored-by: Jonathan Gibbons <jonored@gmail.com>
This commit is contained in:
John Ericson 2024-05-21 17:07:23 -04:00
parent 00d2bf91b2
commit a9fdd537ac
28 changed files with 514 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/libexpr/primops.cc
View file

@ -1605,6 +1605,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drv.structuredAttrs = std::move(*jsonObject);
}
drv.options = derivationOptionsFromStructuredAttrs(*state.store, drv.inputDrvs, drv.env, drv.structuredAttrs);
/* Everything in the context of the strings in the derivation
attributes should be added as dependencies of the resulting
derivation. */

24
src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json
View file

@ -17,6 +17,30 @@
"srcs": []
},
"name": "advanced-attributes-defaults",
"options": {
"additionalSandboxProfile": "",
"allowLocalNetworking": false,
"allowSubstitutes": true,
"exportReferencesGraph": {},
"impureEnvVars": [],
"impureHostDeps": [],
"noChroot": false,
"outputChecks": {
"forAllOutputs": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": true,
"maxClosureSize": null,
"maxSize": null
}
},
"passAsFile": [],
"preferLocalBuild": false,
"requiredSystemFeatures": [],
"unsafeDiscardReferences": {}
},
"outputs": {
"out": {
"hashAlgo": "sha256",

16
src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json
View file

@ -13,6 +13,22 @@
"srcs": []
},
"name": "advanced-attributes-structured-attrs-defaults",
"options": {
"additionalSandboxProfile": "",
"allowLocalNetworking": false,
"allowSubstitutes": true,
"exportReferencesGraph": {},
"impureEnvVars": [],
"impureHostDeps": [],
"noChroot": false,
"outputChecks": {
"perOutput": {}
},
"passAsFile": [],
"preferLocalBuild": false,
"requiredSystemFeatures": [],
"unsafeDiscardReferences": {}
},
"outputs": {
"dev": {
"hashAlgo": "sha256",

66
src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json
View file

@ -31,6 +31,72 @@
]
},
"name": "advanced-attributes-structured-attrs",
"options": {
"additionalSandboxProfile": "sandcastle",
"allowLocalNetworking": true,
"allowSubstitutes": false,
"exportReferencesGraph": {
"refs1": [
"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
],
"refs2": [
"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
]
},
"impureEnvVars": [
"UNICORN"
],
"impureHostDeps": [
"/usr/bin/ditto"
],
"noChroot": true,
"outputChecks": {
"perOutput": {
"bin": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [
"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"
],
"disallowedRequisites": [
"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
],
"ignoreSelfRefs": false,
"maxClosureSize": null,
"maxSize": null
},
"dev": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": false,
"maxClosureSize": 5909,
"maxSize": 789
},
"out": {
"allowedReferences": [
"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
],
"allowedRequisites": [
"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"
],
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": false,
"maxClosureSize": null,
"maxSize": null
}
}
},
"passAsFile": [],
"preferLocalBuild": true,
"requiredSystemFeatures": [
"rainbow",
"uid-range"
],
"unsafeDiscardReferences": {}
},
"outputs": {
"bin": {
"hashAlgo": "sha256",

46
src/libstore-tests/data/derivation/ca/advanced-attributes.json
View file

@ -47,6 +47,52 @@
]
},
"name": "advanced-attributes",
"options": {
"additionalSandboxProfile": "sandcastle",
"allowLocalNetworking": true,
"allowSubstitutes": false,
"exportReferencesGraph": {
"refs1": [
"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
],
"refs2": [
"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
]
},
"impureEnvVars": [
"UNICORN"
],
"impureHostDeps": [
"/usr/bin/ditto"
],
"noChroot": true,
"outputChecks": {
"forAllOutputs": {
"allowedReferences": [
"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
],
"allowedRequisites": [
"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"
],
"disallowedReferences": [
"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"
],
"disallowedRequisites": [
"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
],
"ignoreSelfRefs": true,
"maxClosureSize": null,
"maxSize": null
}
},
"passAsFile": [],
"preferLocalBuild": true,
"requiredSystemFeatures": [
"rainbow",
"uid-range"
],
"unsafeDiscardReferences": {}
},
"outputs": {
"out": {
"hashAlgo": "sha256",

24
src/libstore-tests/data/derivation/dyn-dep-derivation.json
View file

@ -35,6 +35,30 @@
]
},
"name": "dyn-dep-derivation",
"options": {
"additionalSandboxProfile": "",
"allowLocalNetworking": false,
"allowSubstitutes": true,
"exportReferencesGraph": {},
"impureEnvVars": [],
"impureHostDeps": [],
"noChroot": false,
"outputChecks": {
"forAllOutputs": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": true,
"maxClosureSize": null,
"maxSize": null
}
},
"passAsFile": [],
"preferLocalBuild": false,
"requiredSystemFeatures": [],
"unsafeDiscardReferences": {}
},
"outputs": {},
"system": "wasm-sel4",
"version": 4

24
src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json
View file

@ -15,6 +15,30 @@
"srcs": []
},
"name": "advanced-attributes-defaults",
"options": {
"additionalSandboxProfile": "",
"allowLocalNetworking": false,
"allowSubstitutes": true,
"exportReferencesGraph": {},
"impureEnvVars": [],
"impureHostDeps": [],
"noChroot": false,
"outputChecks": {
"forAllOutputs": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": true,
"maxClosureSize": null,
"maxSize": null
}
},
"passAsFile": [],
"preferLocalBuild": false,
"requiredSystemFeatures": [],
"unsafeDiscardReferences": {}
},
"outputs": {
"out": {
"path": "1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"

16
src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json
View file

@ -13,6 +13,22 @@
"srcs": []
},
"name": "advanced-attributes-structured-attrs-defaults",
"options": {
"additionalSandboxProfile": "",
"allowLocalNetworking": false,
"allowSubstitutes": true,
"exportReferencesGraph": {},
"impureEnvVars": [],
"impureHostDeps": [],
"noChroot": false,
"outputChecks": {
"perOutput": {}
},
"passAsFile": [],
"preferLocalBuild": false,
"requiredSystemFeatures": [],
"unsafeDiscardReferences": {}
},
"outputs": {
"dev": {
"path": "8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev"

66
src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json
View file

@ -31,6 +31,72 @@
]
},
"name": "advanced-attributes-structured-attrs",
"options": {
"additionalSandboxProfile": "sandcastle",
"allowLocalNetworking": true,
"allowSubstitutes": false,
"exportReferencesGraph": {
"refs1": [
"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
],
"refs2": [
"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
]
},
"impureEnvVars": [
"UNICORN"
],
"impureHostDeps": [
"/usr/bin/ditto"
],
"noChroot": true,
"outputChecks": {
"perOutput": {
"bin": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [
"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"
],
"disallowedRequisites": [
"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
],
"ignoreSelfRefs": false,
"maxClosureSize": null,
"maxSize": null
},
"dev": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": false,
"maxClosureSize": 5909,
"maxSize": 789
},
"out": {
"allowedReferences": [
"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
],
"allowedRequisites": [
"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
],
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": false,
"maxClosureSize": null,
"maxSize": null
}
}
},
"passAsFile": [],
"preferLocalBuild": true,
"requiredSystemFeatures": [
"rainbow",
"uid-range"
],
"unsafeDiscardReferences": {}
},
"outputs": {
"bin": {
"path": "cnpasdljgkhnwaf78cf3qygcp4qbki1c-advanced-attributes-structured-attrs-bin"

46
src/libstore-tests/data/derivation/ia/advanced-attributes.json
View file

@ -45,6 +45,52 @@
]
},
"name": "advanced-attributes",
"options": {
"additionalSandboxProfile": "sandcastle",
"allowLocalNetworking": true,
"allowSubstitutes": false,
"exportReferencesGraph": {
"refs1": [
"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
],
"refs2": [
"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
]
},
"impureEnvVars": [
"UNICORN"
],
"impureHostDeps": [
"/usr/bin/ditto"
],
"noChroot": true,
"outputChecks": {
"forAllOutputs": {
"allowedReferences": [
"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
],
"allowedRequisites": [
"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
],
"disallowedReferences": [
"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"
],
"disallowedRequisites": [
"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
],
"ignoreSelfRefs": true,
"maxClosureSize": null,
"maxSize": null
}
},
"passAsFile": [],
"preferLocalBuild": true,
"requiredSystemFeatures": [
"rainbow",
"uid-range"
],
"unsafeDiscardReferences": {}
},
"outputs": {
"out": {
"path": "ymqmybkq5j4nd1xplw6ccdpbjnfi017v-advanced-attributes"

24
src/libstore-tests/data/derivation/simple-derivation.json
View file

@ -22,6 +22,30 @@
]
},
"name": "simple-derivation",
"options": {
"additionalSandboxProfile": "",
"allowLocalNetworking": false,
"allowSubstitutes": true,
"exportReferencesGraph": {},
"impureEnvVars": [],
"impureHostDeps": [],
"noChroot": false,
"outputChecks": {
"forAllOutputs": {
"allowedReferences": null,
"allowedRequisites": null,
"disallowedReferences": [],
"disallowedRequisites": [],
"ignoreSelfRefs": true,
"maxClosureSize": null,
"maxSize": null
}
},
"passAsFile": [],
"preferLocalBuild": false,
"requiredSystemFeatures": [],
"unsafeDiscardReferences": {}
},
"outputs": {},
"system": "wasm-sel4",
"version": 4

105
src/libstore-tests/derivation-advanced-attrs.cc
View file

@ -187,17 +187,14 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs(
*this->store, got.inputDrvs, got.env, got.structuredAttrs, true, this->mockXpSettings);
EXPECT_TRUE(!got.structuredAttrs);
EXPECT_EQ(options, advancedAttributes_defaults);
EXPECT_EQ(got.options, advancedAttributes_defaults);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
EXPECT_EQ(got.options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(got.options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(got.options.substitutesAllowed(), true);
EXPECT_EQ(got.options.useUidRange(got), false);
});
};
@ -233,19 +230,16 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
this->readTest("advanced-attributes.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs(
*this->store, got.inputDrvs, got.env, got.structuredAttrs, true, this->mockXpSettings);
EXPECT_TRUE(!got.structuredAttrs);
// Reset fields that vary between test cases to enable whole-object comparison
options.outputChecks = DerivationOptions<SingleDerivedPath>::OutputChecks{.ignoreSelfRefs = true};
options.exportReferencesGraph = {};
got.options.outputChecks = DerivationOptions<SingleDerivedPath>::OutputChecks{.ignoreSelfRefs = true};
got.options.exportReferencesGraph = {};
EXPECT_EQ(options, expected);
EXPECT_EQ(got.options, expected);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
EXPECT_EQ(got.options.substitutesAllowed(), false);
EXPECT_EQ(got.options.useUidRange(got), true);
});
};
@ -334,12 +328,12 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
EXPECT_TRUE(got.structuredAttrs);
EXPECT_EQ(options, advancedAttributes_structuredAttrs_defaults);
EXPECT_EQ(got.options, advancedAttributes_structuredAttrs_defaults);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), true);
EXPECT_EQ(options.useUidRange(got), false);
EXPECT_EQ(got.options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(got.options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(got.options.substitutesAllowed(), true);
EXPECT_EQ(got.options.useUidRange(got), false);
});
};
@ -380,9 +374,6 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs(
*this->store, got.inputDrvs, got.env, got.structuredAttrs, true, this->mockXpSettings);
EXPECT_TRUE(got.structuredAttrs);
// Reset fields that vary between test cases to enable whole-object comparison
@ -390,7 +381,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
// Delete all keys but "dev" in options.outputChecks
auto * outputChecksMapP =
std::get_if<std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>>(
&options.outputChecks);
&got.options.outputChecks);
ASSERT_TRUE(outputChecksMapP);
auto & outputChecksMap = *outputChecksMapP;
auto devEntry = outputChecksMap.find("dev");
@ -399,14 +390,14 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
outputChecksMap.clear();
outputChecksMap.emplace("dev", std::move(devChecks));
}
options.exportReferencesGraph = {};
got.options.exportReferencesGraph = {};
EXPECT_EQ(options, expected);
EXPECT_EQ(got.options, expected);
EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(options.substitutesAllowed(), false);
EXPECT_EQ(options.useUidRange(got), true);
EXPECT_EQ(got.options.canBuildLocally(*this->store, got), false);
EXPECT_EQ(got.options.willBuildLocally(*this->store, got), false);
EXPECT_EQ(got.options.substitutesAllowed(), false);
EXPECT_EQ(got.options.useUidRange(got), true);
});
};
@ -517,4 +508,56 @@ TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, structuredAttrs_all_set, struct
#undef TEST_JSON_OPTIONS
#define SYNC_CONFLICT(NAME, VALUE) \
NAME = VALUE; \
EXPECT_THROW(got.unparse(*store, false), Error); \
got.options = options;
TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_option_syncConflict)
{
readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
auto got = parseDerivation(*store, std::move(encoded), "foo");
auto options = got.options;
SYNC_CONFLICT(got.options.additionalSandboxProfile, "foobar");
SYNC_CONFLICT(got.options.noChroot, true);
SYNC_CONFLICT(got.options.impureHostDeps, StringSet{"/usr/bin/ditto"});
SYNC_CONFLICT(got.options.impureEnvVars, StringSet{"HELLO"});
SYNC_CONFLICT(got.options.allowLocalNetworking, true);
SYNC_CONFLICT(std::get<0>(got.options.outputChecks).allowedReferences, StringSet{"nothing"});
SYNC_CONFLICT(std::get<0>(got.options.outputChecks).allowedRequisites, StringSet{"hey"});
SYNC_CONFLICT(std::get<0>(got.options.outputChecks).disallowedReferences, StringSet{"BAR"});
SYNC_CONFLICT(std::get<0>(got.options.outputChecks).disallowedRequisites, StringSet{"FOO"});
});
};
#undef SYNC_CONFLICT
#define SYNC_CONFLICT(NAME, VALUE) \
got.env[NAME] = VALUE; \
EXPECT_THROW(got.unparse(*store, false), Error); \
got.env = env;
TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_env_syncConflict)
{
readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
auto got = parseDerivation(*store, std::move(encoded), "foo");
auto env = got.env;
// TODO: Is there any way to serialize a boolean/StringSet into an env value (string)?
// Something like `State::coerceToString`
SYNC_CONFLICT("__sandboxProfile", "foobar");
SYNC_CONFLICT("__noChroot", "1");
SYNC_CONFLICT("__impureHostDeps", "/usr/bin/ditto");
SYNC_CONFLICT("impureEnvVars", "FOOBAR");
SYNC_CONFLICT("__darwinAllowLocalNetworking", "1");
SYNC_CONFLICT("allowedReferences", "nothing");
SYNC_CONFLICT("allowedRequisites", "hey");
SYNC_CONFLICT("disallowedReferences", "BAR");
SYNC_CONFLICT("disallowedRequisites", "FOO");
});
};
#undef SYNC_CONFLICT
} // namespace nix

4
src/libstore-tests/derivation.cc
View file

@ -254,6 +254,8 @@ Derivation makeSimpleDrv()
"WOLF",
},
};
drv.options =
derivationOptionsFromStructuredAttrs(StoreDirConfig{"/nix/store"}, drv.inputDrvs, drv.env, drv.structuredAttrs);
return drv;
}
@ -321,6 +323,8 @@ Derivation makeDynDepDerivation()
"WOLF",
},
};
drv.options =
derivationOptionsFromStructuredAttrs(StoreDirConfig{"/nix/store"}, drv.inputDrvs, drv.env, drv.structuredAttrs);
return drv;
}

2
src/libstore-tests/write-derivation.cc
View file

@ -43,6 +43,8 @@ static Derivation makeSimpleDrv()
TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce)
{
auto drv = makeSimpleDrv();
drv.options =
derivationOptionsFromStructuredAttrs(StoreDirConfig{"/nix/store"}, drv.inputDrvs, drv.env, drv.structuredAttrs);
auto path1 = writeDerivation(*store, drv, NoRepair);
config->readOnly = true;

19
src/libstore/build/derivation-building-goal.cc
View file

@ -32,16 +32,9 @@ DerivationBuildingGoal::DerivationBuildingGoal(
, drv{std::make_unique<Derivation>(drv)}
, buildMode(buildMode)
{
DerivationOptions<SingleDerivedPath> temp;
try {
temp = derivationOptionsFromStructuredAttrs(worker.store, drv.inputDrvs, drv.env, drv.structuredAttrs);
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
auto x = tryResolve(
temp, [&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
drv.options,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
try {
return resolveDerivedPath(
worker.store, SingleDerivedPath::Built{drvPath, outputName}, &worker.evalStore);
@ -357,7 +350,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
bool buildLocally = (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
bool buildLocally = (buildMode != bmNormal || drv->options.willBuildLocally(worker.store, *drv))
&& settings.maxBuildJobs.get() != 0;
if (buildLocally) {
@ -392,7 +385,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv);
if (!externalBuilder && !drvOptions->canBuildLocally(worker.store, *drv)) {
if (!externalBuilder && !drv->options.canBuildLocally(worker.store, *drv)) {
auto msg =
fmt("Cannot build '%s'.\n"
"Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL
@ -401,7 +394,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
"Current system: '%s' with features {%s}",
Magenta(worker.store.printStorePath(drvPath)),
Magenta(drv->platform),
concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
concatStringsSep(", ", drv->options.getRequiredSystemFeatures(*drv)),
Magenta(settings.thisSystem),
concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures));
@ -833,7 +826,7 @@ HookReply DerivationBuildingGoal::tryBuildHook(const std::map<std::string, Initi
/* Send the request to the hook. */
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
<< worker.store.printStorePath(drvPath) << drvOptions->getRequiredSystemFeatures(*drv);
<< worker.store.printStorePath(drvPath) << drv->options.getRequiredSystemFeatures(*drv);
worker.hook->sink.flush();
/* Read the first line of input, which should be a word indicating

13
src/libstore/build/derivation-goal.cc
View file

@ -64,15 +64,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
{
trace("have derivation");
auto drvOptions = [&]() -> DerivationOptions<SingleDerivedPath> {
try {
return derivationOptionsFromStructuredAttrs(worker.store, drv->inputDrvs, drv->env, drv->structuredAttrs);
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
}();
if (!drv->type().hasKnownOutputPaths())
experimentalFeatureSettings.require(Xp::CaDerivations);
@ -98,7 +89,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (settings.useSubstitutes && drv->options.substitutesAllowed()) {
if (!checkResult)
waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput})));
else {
@ -151,7 +142,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
}
if (resolutionGoal->resolvedDrv) {
auto & [pathResolved, drvResolved] = *resolutionGoal->resolvedDrv;
auto & [pathResolved, drvResolved, drvOptionsResolved] = *resolutionGoal->resolvedDrv;
auto resolvedDrvGoal =
worker.makeDerivationGoal(pathResolved, drvResolved, wantedOutput, buildMode, /*storeDerivation=*/true);

7
src/libstore/build/derivation-resolution-goal.cc
View file

@ -164,7 +164,8 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
}
assert(attempt);
auto pathResolved = writeDerivation(worker.store, *attempt, NoRepair, /*readOnly =*/true);
// TODO check options compatibility with ATerm.
auto pathResolved = writeDerivation(worker.store, attempt->first, NoRepair, /*readOnly =*/true);
auto msg =
fmt("resolved derivation: '%s' -> '%s'",
@ -180,8 +181,8 @@ Goal::Co DerivationResolutionGoal::resolveDerivation()
worker.store.printStorePath(pathResolved),
});
resolvedDrv =
std::make_unique<std::pair<StorePath, BasicDerivation>>(std::move(pathResolved), *std::move(attempt));
resolvedDrv = std::make_unique<std::tuple<StorePath, BasicDerivation, DerivationOptions<StorePath>>>(
std::move(pathResolved), std::move(attempt->first), std::move(attempt->second));
}
}

12
src/libstore/derivation-options.cc
View file

@ -250,21 +250,11 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
return {
.outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> {
if (parsed) {
auto & structuredAttrs = parsed->structuredAttrs;
std::map<std::string, OutputChecks<SingleDerivedPath>> res;
if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
if (auto * outputChecks = get(parsed->structuredAttrs, "outputChecks")) {
for (auto & [outputName, output_] : getObject(*outputChecks)) {
OutputChecks<SingleDerivedPath> checks;
auto & output = getObject(output_);
if (auto maxSize = get(output, "maxSize"))
checks.maxSize = maxSize->get<uint64_t>();
if (auto maxClosureSize = get(output, "maxClosureSize"))
checks.maxClosureSize = maxClosureSize->get<uint64_t>();
auto get_ =
[&](const std::string & name) -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
if (auto i = get(output, name)) {

29
src/libstore/derivations.cc
View file

@ -508,6 +508,9 @@ Derivation parseDerivation(
}
expect(str, ')');
drv.options = derivationOptionsFromStructuredAttrs(store, drv.inputDrvs, drv.env, drv.structuredAttrs);
return drv;
}
@ -634,6 +637,14 @@ static bool hasDynamicDrvDep(const Derivation & drv)
std::string Derivation::unparse(
const StoreDirConfig & store, bool maskOutputs, DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
{
{
auto optionsFromEnv = derivationOptionsFromStructuredAttrs(store, inputDrvs, env, structuredAttrs);
if (optionsFromEnv != options)
throw Error(
"'drv.options' and 'drv.env' are out of sync. This is probably an internal error, please open an issue!");
}
std::string s;
s.reserve(65536);
@ -1127,7 +1138,8 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
std::optional<std::pair<BasicDerivation, DerivationOptions<StorePath>>>
Derivation::tryResolve(Store & store, Store * evalStore) const
{
return tryResolve(
store, [&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
@ -1184,7 +1196,7 @@ static bool tryResolveInput(
return true;
}
std::optional<BasicDerivation> Derivation::tryResolve(
std::optional<std::pair<BasicDerivation, DerivationOptions<StorePath>>> Derivation::tryResolve(
Store & store,
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
queryResolutionChain) const
@ -1207,7 +1219,12 @@ std::optional<BasicDerivation> Derivation::tryResolve(
rewriteDerivation(store, resolved, inputRewrites);
return resolved;
auto resolvedOptions = nix::tryResolve(options, queryResolutionChain);
if (!resolvedOptions)
return std::nullopt;
return {{std::move(resolved), *std::move(resolvedOptions)}};
}
void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
@ -1431,6 +1448,7 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
res["builder"] = d.builder;
res["args"] = d.args;
res["env"] = d.env;
res["options"] = d.options;
if (d.structuredAttrs)
res["structuredAttrs"] = d.structuredAttrs->structuredAttrs;
@ -1511,6 +1529,11 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
if (auto structuredAttrs = get(json, "structuredAttrs"))
res.structuredAttrs = StructuredAttrs{*structuredAttrs};
if (auto options = get(json, "options"))
res.options = *options;
else
res.options = derivationOptionsFromStructuredAttrs(store, res.inputDrvs, res.env, res.structuredAttrs);
return res;
}

3
src/libstore/include/nix/store/build/derivation-builder.hh
View file

@ -67,7 +67,8 @@ struct DerivationBuilderParams
/**
* The derivation options of `drv`.
*
* @todo this should be part of `Derivation`.
* @todo this should be part of `Derivation`/`BasicDerivation`, if
* those two were distinguished by type arguments not subtyping.
*/
const DerivationOptions<StorePath> & drvOptions;

1
src/libstore/include/nix/store/build/derivation-goal.hh
View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "nix/store/parsed-derivations.hh"
#include "nix/store/derivations.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/build/derivation-building-misc.hh"

2
src/libstore/include/nix/store/build/derivation-resolution-goal.hh
View file

@ -41,7 +41,7 @@ struct DerivationResolutionGoal : public Goal
* If the derivation needed to be resolved, this is resulting
* resolved derivations and its path.
*/
std::unique_ptr<std::pair<StorePath, BasicDerivation>> resolvedDrv;
std::unique_ptr<std::tuple<StorePath, BasicDerivation, DerivationOptions<StorePath>>> resolvedDrv;
void timedOut(Error && ex) override {}

7
src/libstore/include/nix/store/derivation-options.hh
View file

@ -247,6 +247,13 @@ std::optional<DerivationOptions<StorePath>> tryResolve(
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
queryResolutionChain);
template<typename T>
struct json_avoids_null;
template<typename Input>
struct json_avoids_null<DerivationOptions<Input>> : std::true_type
{};
}; // namespace nix
JSON_IMPL(nix::DerivationOptions<nix::StorePath>);

16
src/libstore/include/nix/store/derivations.hh
View file

@ -8,6 +8,7 @@
#include "nix/util/repair-flag.hh"
#include "nix/store/derived-path-map.hh"
#include "nix/store/parsed-derivations.hh"
#include "nix/store/derivation-options.hh"
#include "nix/util/sync.hh"
#include "nix/util/variant-wrapper.hh"
@ -332,6 +333,16 @@ struct Derivation : BasicDerivation
*/
DerivedPathMap<std::set<OutputName, std::less<>>> inputDrvs;
/**
* Derivation options
*
* @todo instead of `BasicDerivation`/`Derivation`, should just have
* `template<...> Derivation`, and then the choice of template
* parameter would control "possibly-unresolved vs definitely
* resolved" and this field would use the overall type parameter.
*/
DerivationOptions<SingleDerivedPath> options;
/**
* Print a derivation.
*/
@ -349,14 +360,15 @@ struct Derivation : BasicDerivation
* 2. Input placeholders are replaced with realized input store
* paths.
*/
std::optional<BasicDerivation> tryResolve(Store & store, Store * evalStore = nullptr) const;
std::optional<std::pair<BasicDerivation, DerivationOptions<StorePath>>>
tryResolve(Store & store, Store * evalStore = nullptr) const;
/**
* Like the above, but instead of querying the Nix database for
* realisations, uses a given mapping from input derivation paths +
* output names to actual output store paths.
*/
std::optional<BasicDerivation> tryResolve(
std::optional<std::pair<BasicDerivation, DerivationOptions<StorePath>>> tryResolve(
Store & store,
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
queryResolutionChain) const;

14
src/libstore/misc.cc
View file

@ -224,18 +224,8 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
return;
auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
DerivationOptions<SingleDerivedPath> drvOptions;
try {
// FIXME: this is a lot of work just to get the value
// of `allowSubstitutes`.
drvOptions =
derivationOptionsFromStructuredAttrs(*this, drv->inputDrvs, drv->env, drv->structuredAttrs);
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath));
throw;
}
if (!knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (!knownOutputPaths && settings.useSubstitutes && drv->options.substitutesAllowed()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
// If there are unknown output paths, attempt to find if the
@ -265,7 +255,7 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
}
}
if (knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (knownOutputPaths && settings.useSubstitutes && drv->options.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState));

6
src/libstore/store-api.cc
View file

@ -1169,8 +1169,10 @@ std::optional<StorePath> Store::getBuildDerivationPath(const StorePath & path)
// The build log is actually attached to the corresponding
// resolved derivation, so we need to get it first
auto resolvedDrv = drv.tryResolve(*this);
if (resolvedDrv)
return ::nix::writeDerivation(*this, *resolvedDrv, NoRepair, true);
if (resolvedDrv) {
// TODO check options compatibility with ATerm.
return ::nix::writeDerivation(*this, resolvedDrv->first, NoRepair, true);
}
}
return path;

18
src/nix/nix-build/nix-build.cc
View file

@ -532,9 +532,9 @@ static void main_nix_build(int argc, char ** argv)
}
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
auto resolvedDrv = drv.tryResolve(*store);
assert(resolvedDrv && "Successfully resolved the derivation");
drv = *resolvedDrv;
auto resolved = drv.tryResolve(*store);
assert(resolved && "Successfully resolved the derivation");
drv = resolved->first;
}
// Set the environment.
@ -554,18 +554,10 @@ static void main_nix_build(int argc, char ** argv)
env["NIX_STORE"] = store->storeDir;
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
DerivationOptions<StorePath> drvOptions;
try {
drvOptions = derivationOptionsFromStructuredAttrs(*store, drv.env, drv.structuredAttrs);
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath()));
throw;
}
int fileNr = 0;
for (auto & var : drv.env)
if (drvOptions.passAsFile.count(var.first)) {
if (drv.options.passAsFile.count(var.first)) {
auto fn = ".attr-" + std::to_string(fileNr++);
Path p = (tmpDir.path() / fn).string();
writeFile(p, var.second);
@ -594,7 +586,7 @@ static void main_nix_build(int argc, char ** argv)
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
accumInputClosure(inputDrv, inputNode);
auto json = drv.structuredAttrs->prepareStructuredAttrs(*store, drvOptions, inputs, drv.outputs);
auto json = drv.structuredAttrs->prepareStructuredAttrs(*store, drv.options, inputs, drv.outputs);
structuredAttrsRC = StructuredAttrs::writeShell(json);

6
tests/functional/lang/eval-okay-derivation-legacy.err.exp
View file

@ -4,3 +4,9 @@ warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs'
warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks.<output>.disallowedRequisites' instead
warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead
warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks.<output>.maxSize' instead
warning: 'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead
warning: 'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead
warning: 'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead
warning: 'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead
warning: 'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead
warning: 'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead