1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-15 23:12:44 +01:00

Change JSON derivation format in two ways

- Use canonical content address JSON format for floating content
  addressed derivation outputs

  This keeps it more consistent.

- Reorganize inputs into nested structure (`inputs.srcs` and
  `inputs.drvs`)

  This will allow for an easier to use, but less compact, alternative
  where `srcs` is just a list of derived paths.

  It also allows for other experiments for derivations with a different
  input structure, as I suspect will be needed for secure build traces.
This commit is contained in:
John Ericson 2025-10-13 00:24:12 -04:00
parent 147e183c68
commit 0c37a62207
21 changed files with 298 additions and 251 deletions

View file

@ -70,7 +70,7 @@ schemas += [
# Match overall
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml',
'schema' : schema_dir / 'derivation-v4.yaml',
'files' : [
'dyn-dep-derivation.json',
'simple-derivation.json',
@ -78,7 +78,7 @@ schemas += [
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/overall',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/overall',
'files' : [
'output-caFixedFlat.json',
'output-caFixedNAR.json',
@ -92,14 +92,14 @@ schemas += [
# Match exact variant
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/inputAddressed',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/inputAddressed',
'files' : [
'output-inputAddressed.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFixed',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFixed',
'files' : [
'output-caFixedFlat.json',
'output-caFixedNAR.json',
@ -108,21 +108,21 @@ schemas += [
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFloating',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFloating',
'files' : [
'output-caFloating.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/deferred',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/deferred',
'files' : [
'output-deferred.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/impure',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/impure',
'files' : [
'output-impure.json',
],

View file

@ -12,8 +12,10 @@
"outputHashMode": "recursive",
"system": "my-system"
},
"inputDrvs": {},
"inputSrcs": [],
"inputs": {
"drvs": {},
"srcs": []
},
"name": "advanced-attributes-defaults",
"outputs": {
"out": {
@ -22,5 +24,5 @@
}
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -8,8 +8,10 @@
"dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
"out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
},
"inputDrvs": {},
"inputSrcs": [],
"inputs": {
"drvs": {},
"srcs": []
},
"name": "advanced-attributes-structured-attrs-defaults",
"outputs": {
"dev": {
@ -33,5 +35,5 @@
"system": "my-system"
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -9,25 +9,27 @@
"dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
"out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
},
"inputDrvs": {
"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
"inputs": {
"drvs": {
"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
},
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
},
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
"srcs": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
]
},
"inputSrcs": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
],
"name": "advanced-attributes-structured-attrs",
"outputs": {
"bin": {
@ -101,5 +103,5 @@
"system": "my-system"
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -25,25 +25,27 @@
"requiredSystemFeatures": "rainbow uid-range",
"system": "my-system"
},
"inputDrvs": {
"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
"inputs": {
"drvs": {
"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
},
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
},
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
"srcs": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
]
},
"inputSrcs": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
],
"name": "advanced-attributes",
"outputs": {
"out": {
@ -52,5 +54,5 @@
}
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -10,8 +10,10 @@
"out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9",
"system": "x86_64-linux"
},
"inputDrvs": {},
"inputSrcs": [],
"inputs": {
"drvs": {},
"srcs": []
},
"name": "myname",
"outputs": {
"out": {
@ -20,5 +22,5 @@
}
},
"system": "x86_64-linux",
"version": 3
"version": 4
}

View file

@ -7,33 +7,35 @@
"env": {
"BIG_BAD": "WOLF"
},
"inputDrvs": {
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {
"cat": {
"dynamicOutputs": {},
"outputs": [
"kitten"
]
"inputs": {
"drvs": {
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {
"cat": {
"dynamicOutputs": {},
"outputs": [
"kitten"
]
},
"goose": {
"dynamicOutputs": {},
"outputs": [
"gosling"
]
}
},
"goose": {
"dynamicOutputs": {},
"outputs": [
"gosling"
]
}
},
"outputs": [
"cat",
"dog"
]
}
"outputs": [
"cat",
"dog"
]
}
},
"srcs": [
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
]
},
"inputSrcs": [
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
],
"name": "dyn-dep-derivation",
"outputs": {},
"system": "wasm-sel4",
"version": 3
"version": 4
}

View file

@ -10,8 +10,10 @@
"out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults",
"system": "my-system"
},
"inputDrvs": {},
"inputSrcs": [],
"inputs": {
"drvs": {},
"srcs": []
},
"name": "advanced-attributes-defaults",
"outputs": {
"out": {
@ -19,5 +21,5 @@
}
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -8,8 +8,10 @@
"dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev",
"out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults"
},
"inputDrvs": {},
"inputSrcs": [],
"inputs": {
"drvs": {},
"srcs": []
},
"name": "advanced-attributes-structured-attrs-defaults",
"outputs": {
"dev": {
@ -29,5 +31,5 @@
"system": "my-system"
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -9,25 +9,27 @@
"dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev",
"out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
},
"inputDrvs": {
"afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
"inputs": {
"drvs": {
"afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
},
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
},
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
"srcs": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
]
},
"inputSrcs": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
],
"name": "advanced-attributes-structured-attrs",
"outputs": {
"bin": {
@ -96,5 +98,5 @@
"system": "my-system"
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -23,25 +23,27 @@
"requiredSystemFeatures": "rainbow uid-range",
"system": "my-system"
},
"inputDrvs": {
"afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
"inputs": {
"drvs": {
"afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
},
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
},
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
"dynamicOutputs": {},
"outputs": [
"dev",
"out"
]
}
"srcs": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
]
},
"inputSrcs": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
],
"name": "advanced-attributes",
"outputs": {
"out": {
@ -49,5 +51,5 @@
}
},
"system": "my-system",
"version": 3
"version": 4
}

View file

@ -1,5 +1,8 @@
{
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"hashAlgo": "sha256",
"hash": {
"algorithm": "sha256",
"format": "base64",
"hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="
},
"method": "flat"
}

View file

@ -1,5 +1,8 @@
{
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"hashAlgo": "sha256",
"hash": {
"algorithm": "sha256",
"format": "base64",
"hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="
},
"method": "nar"
}

View file

@ -1,5 +1,8 @@
{
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"hashAlgo": "sha256",
"hash": {
"algorithm": "sha256",
"format": "base64",
"hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="
},
"method": "text"
}

View file

@ -7,20 +7,22 @@
"env": {
"BIG_BAD": "WOLF"
},
"inputDrvs": {
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {},
"outputs": [
"cat",
"dog"
]
}
"inputs": {
"drvs": {
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {},
"outputs": [
"cat",
"dog"
]
}
},
"srcs": [
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
]
},
"inputSrcs": [
"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
],
"name": "simple-derivation",
"outputs": {},
"system": "wasm-sel4",
"version": 3
"version": 4
}

View file

@ -636,7 +636,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
auto outj_ph = nix::hashPlaceholder("outj");
std::string drvJson = R"({
"version": 3,
"version": 4,
"name": "multi-output-test",
"system": ")" + nix::settings.thisSystem.get()
+ R"(",
@ -668,8 +668,10 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
"outa": ")" + outa_ph
+ R"("
},
"inputDrvs": {},
"inputSrcs": [],
"inputs": {
"drvs": {},
"srcs": []
},
"outputs": {
"outd": { "hashAlgo": "sha256", "method": "nar" },
"outf": { "hashAlgo": "sha256", "method": "nar" },

View file

@ -1293,15 +1293,13 @@ void adl_serializer<DerivationOutput>::to_json(json & res, const DerivationOutpu
overloaded{
[&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; },
[&](const DerivationOutput::CAFixed & dof) {
/* it would be nice to output the path for user convenience, but
this would require us to know the store dir. */
res = dof.ca;
// FIXME print refs?
/* it would be nice to output the path for user convenience, but
this would require us to know the store dir. */
#if 0
res["path"] = dof.path(store, drvName, outputName);
#endif
res["method"] = std::string{dof.ca.method.render()};
res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo);
res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false);
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {
res["method"] = std::string{dof.method.render()};
@ -1341,15 +1339,12 @@ adl_serializer<DerivationOutput>::from_json(const json & _json, const Experiment
};
}
else if (keys == (std::set<std::string_view>{"method", "hashAlgo", "hash"})) {
auto [method, hashAlgo] = methodAlgo();
else if (keys == (std::set<std::string_view>{"method", "hash"})) {
auto dof = DerivationOutput::CAFixed{
.ca =
ContentAddress{
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo),
},
.ca = static_cast<ContentAddress>(_json),
};
if (dof.ca.method == ContentAddressMethod::Raw::Text)
xpSettings.require(Xp::DynamicDerivations, "text-hashed derivation output in JSON");
/* We no longer produce this (denormalized) field (for the
reasons described above), so we don't need to check it. */
#if 0
@ -1392,7 +1387,7 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
res["name"] = d.name;
res["version"] = 3;
res["version"] = 4;
{
nlohmann::json & outputsObj = res["outputs"];
@ -1403,13 +1398,16 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
}
{
auto & inputsList = res["inputSrcs"];
inputsList = nlohmann::json::array();
for (auto & input : d.inputSrcs)
inputsList.emplace_back(input);
}
auto & inputsObj = res["inputs"];
inputsObj = nlohmann::json::object();
{
auto & inputsList = inputsObj["srcs"];
inputsList = nlohmann::json::array();
for (auto & input : d.inputSrcs)
inputsList.emplace_back(input);
}
{
auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json {
auto value = nlohmann::json::object();
value["outputs"] = inputNode.value;
@ -1421,12 +1419,11 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
}
return value;
};
{
auto & inputDrvsObj = res["inputDrvs"];
inputDrvsObj = nlohmann::json::object();
for (auto & [inputDrv, inputNode] : d.inputDrvs.map) {
inputDrvsObj[inputDrv.to_string()] = doInput(inputNode);
}
auto & inputDrvsObj = inputsObj["drvs"];
inputDrvsObj = nlohmann::json::object();
for (auto & [inputDrv, inputNode] : d.inputDrvs.map) {
inputDrvsObj[inputDrv.to_string()] = doInput(inputNode);
}
}
@ -1449,8 +1446,8 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
res.name = getString(valueAt(json, "name"));
if (valueAt(json, "version") != 3)
throw Error("Only derivation format version 3 is currently supported.");
if (valueAt(json, "version") != 4)
throw Error("Only derivation format version 4 is currently supported.");
try {
auto outputs = getObject(valueAt(json, "outputs"));
@ -1463,32 +1460,39 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
}
try {
auto inputSrcs = getArray(valueAt(json, "inputSrcs"));
for (auto & input : inputSrcs)
res.inputSrcs.insert(input);
} catch (Error & e) {
e.addTrace({}, "while reading key 'inputSrcs'");
throw;
}
auto inputsObj = getObject(valueAt(json, "inputs"));
try {
auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap<StringSet>::ChildNode {
auto & json = getObject(_json);
DerivedPathMap<StringSet>::ChildNode node;
node.value = getStringSet(valueAt(json, "outputs"));
auto drvs = getObject(valueAt(json, "dynamicOutputs"));
for (auto & [outputId, childNode] : drvs) {
xpSettings.require(
Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); });
node.childMap[outputId] = doInput(childNode);
}
return node;
};
auto drvs = getObject(valueAt(json, "inputDrvs"));
for (auto & [inputDrvPath, inputOutputs] : drvs)
res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs);
try {
auto inputSrcs = getArray(valueAt(inputsObj, "srcs"));
for (auto & input : inputSrcs)
res.inputSrcs.insert(input);
} catch (Error & e) {
e.addTrace({}, "while reading key 'srcs'");
throw;
}
try {
auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap<StringSet>::ChildNode {
auto & json = getObject(_json);
DerivedPathMap<StringSet>::ChildNode node;
node.value = getStringSet(valueAt(json, "outputs"));
auto drvs = getObject(valueAt(json, "dynamicOutputs"));
for (auto & [outputId, childNode] : drvs) {
xpSettings.require(
Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); });
node.childMap[outputId] = doInput(childNode);
}
return node;
};
auto drvs = getObject(valueAt(inputsObj, "drvs"));
for (auto & [inputDrvPath, inputOutputs] : drvs)
res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs);
} catch (Error & e) {
e.addTrace({}, "while reading key 'drvs'");
throw;
}
} catch (Error & e) {
e.addTrace({}, "while reading key 'inputDrvs'");
e.addTrace({}, "while reading key 'inputs'");
throw;
}