Change JSON derivation format in two ways

- Use canonical content address JSON format for floating content
  addressed derivation outputs

  This keeps it more consistent.

- Reorganize inputs into nested structure (`inputs.srcs` and
  `inputs.drvs`)

  This will allow for an easier to use, but less compact, alternative
  where `srcs` is just a list of derived paths.

  It also allows for other experiments for derivations with a different
  input structure, as I suspect will be needed for secure build traces.

doc/manual/source/protocols/json/derivation.md

@@ -1,7 +1,7 @@
-{{#include derivation-v3-fixed.md}}
+{{#include derivation-v4-fixed.md}}
 
 <!-- need to convert YAML to JSON first
 ## Raw Schema
 
-[JSON Schema for Derivation v3](schema/derivation-v3.json)
+[JSON Schema for Derivation v3](schema/derivation-v4.json)
 -->

doc/manual/source/protocols/json/meson.build

@@ -13,7 +13,7 @@ schemas = [
   'content-address-v1',
   'store-path-v1',
   'store-object-info-v1',
-  'derivation-v3',
+  'derivation-v4',
   'deriving-path-v1',
   'build-trace-entry-v1',
   'build-result-v1',

doc/manual/source/protocols/json/schema/derivation-v4.yaml

@@ -1,8 +1,8 @@
 "$schema": "http://json-schema.org/draft-04/schema"
-"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v4.json"
 title: Derivation
 description: |
-  Experimental JSON representation of a Nix derivation (version 3).
+  Experimental JSON representation of a Nix derivation (version 4).
 
   This schema describes the JSON representation of Nix's `Derivation` type.
 
@@ -17,8 +17,7 @@ required:
   - name
   - version
   - outputs
-  - inputSrcs
+  - inputs
-  - inputDrvs
   - system
   - builder
   - args
@@ -32,10 +31,10 @@ properties:
       Used when calculating store paths for the derivation’s outputs.
 
   version:
-    const: 3
+    const: 4
-    title: Format version (must be 3)
+    title: Format version (must be 4)
     description: |
-      Must be `3`.
+      Must be `4`.
       This is a guard that allows us to continue evolving this format.
       The choice of `3` is fairly arbitrary, but corresponds to this informal version:
 
@@ -47,6 +46,12 @@ properties:
 
       - Version 3: Drop store dir from store paths, just include base name.
 
+      - Version 4: Two cleanups, batched together to lesson churn:
+
+        - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`)
+
+        - Use canonical content address JSON format for floating content addressed derivation outputs.
+
       Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
 
   outputs:
@@ -70,7 +75,16 @@ properties:
     additionalProperties:
       "$ref": "#/$defs/output/overall"
 
-  inputSrcs:
+  inputs:
+    type: object
+    title: Derivation inputs
+    description: |
+      Input dependencies for the derivation, organized into source paths and derivation dependencies.
+    required:
+      - srcs
+      - drvs
+    properties:
+      srcs:
         type: array
         title: Input source paths
         description: |
@@ -79,15 +93,14 @@ properties:
           > **Example**
           >
           > ```json
-      > "inputSrcs": [
+          > "srcs": [
           >   "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
           >   "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
           > ]
           > ```
         items:
           $ref: "store-path-v1.yaml"
-
+      drvs:
-  inputDrvs:
         type: object
         title: Input derivations
         description: |
@@ -96,7 +109,7 @@ properties:
           > **Example**
           >
           > ```json
-      > "inputDrvs": {
+          > "drvs": {
           >   "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
           >   "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
           > }
@@ -112,6 +125,7 @@ properties:
             - "$ref": "#/$defs/outputNames"
             - "$ref": "#/$defs/dynamicOutputs"
         additionalProperties: false
+    additionalProperties: false
 
   system:
     type: string
@@ -189,24 +203,18 @@ properties:
         The output is content-addressed, and the content-address is fixed in advance.
 
         See [Fixed-output content-addressing](@docroot@/store/derivation/outputs/content-address.md#fixed) for more details.
-      type: object
+      "$ref": "./content-address-v1.yaml"
       required:
         - method
-        - hashAlgo
         - hash
       properties:
         method:
-          "$ref": "./content-address-v1.yaml#/$defs/method"
           description: |
             Method of content addressing used for this output.
-        hashAlgo:
-          title: Hash algorithm
-          "$ref": "./hash-v1.yaml#/$defs/algorithm"
         hash:
-          type: string
           title: Expected hash value
           description: |
-            The expected content hash in base-16.
+            The expected content hash.
       additionalProperties: false
 
     caFloating:

src/json-schema-checks/meson.build

@@ -70,7 +70,7 @@ schemas += [
   # Match overall
   {
     'stem' : 'derivation',
-    'schema' : schema_dir / 'derivation-v3.yaml',
+    'schema' : schema_dir / 'derivation-v4.yaml',
     'files' : [
       'dyn-dep-derivation.json',
       'simple-derivation.json',
@@ -78,7 +78,7 @@ schemas += [
   },
   {
     'stem' : 'derivation',
-    'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/overall',
+    'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/overall',
     'files' : [
       'output-caFixedFlat.json',
       'output-caFixedNAR.json',
@@ -92,14 +92,14 @@ schemas += [
   # Match exact variant
   {
     'stem' : 'derivation',
-    'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/inputAddressed',
+    'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/inputAddressed',
     'files' : [
       'output-inputAddressed.json',
     ],
   },
   {
     'stem' : 'derivation',
-    'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFixed',
+    'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFixed',
     'files' : [
       'output-caFixedFlat.json',
       'output-caFixedNAR.json',
@@ -108,21 +108,21 @@ schemas += [
   },
   {
     'stem' : 'derivation',
-    'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFloating',
+    'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFloating',
     'files' : [
       'output-caFloating.json',
     ],
   },
   {
     'stem' : 'derivation',
-    'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/deferred',
+    'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/deferred',
     'files' : [
       'output-deferred.json',
     ],
   },
   {
     'stem' : 'derivation',
-    'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/impure',
+    'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/impure',
     'files' : [
       'output-impure.json',
     ],

src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json

@@ -12,8 +12,10 @@
     "outputHashMode": "recursive",
     "system": "my-system"
   },
-  "inputDrvs": {},
+  "inputs": {
-  "inputSrcs": [],
+    "drvs": {},
+    "srcs": []
+  },
   "name": "advanced-attributes-defaults",
   "outputs": {
     "out": {
@@ -22,5 +24,5 @@
     }
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json

@@ -8,8 +8,10 @@
     "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
   },
-  "inputDrvs": {},
+  "inputs": {
-  "inputSrcs": [],
+    "drvs": {},
+    "srcs": []
+  },
   "name": "advanced-attributes-structured-attrs-defaults",
   "outputs": {
     "dev": {
@@ -33,5 +35,5 @@
     "system": "my-system"
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json

@@ -9,7 +9,8 @@
     "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
   },
-  "inputDrvs": {
+  "inputs": {
+    "drvs": {
       "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
         "dynamicOutputs": {},
         "outputs": [
@@ -25,9 +26,10 @@
         ]
       }
     },
-  "inputSrcs": [
+    "srcs": [
       "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
-  ],
+    ]
+  },
   "name": "advanced-attributes-structured-attrs",
   "outputs": {
     "bin": {
@@ -101,5 +103,5 @@
     "system": "my-system"
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ca/advanced-attributes.json

@@ -25,7 +25,8 @@
     "requiredSystemFeatures": "rainbow uid-range",
     "system": "my-system"
   },
-  "inputDrvs": {
+  "inputs": {
+    "drvs": {
       "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
         "dynamicOutputs": {},
         "outputs": [
@@ -41,9 +42,10 @@
         ]
       }
     },
-  "inputSrcs": [
+    "srcs": [
       "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
-  ],
+    ]
+  },
   "name": "advanced-attributes",
   "outputs": {
     "out": {
@@ -52,5 +54,5 @@
     }
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ca/self-contained.json

@@ -10,8 +10,10 @@
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9",
     "system": "x86_64-linux"
   },
-  "inputDrvs": {},
+  "inputs": {
-  "inputSrcs": [],
+    "drvs": {},
+    "srcs": []
+  },
   "name": "myname",
   "outputs": {
     "out": {
@@ -20,5 +22,5 @@
     }
   },
   "system": "x86_64-linux",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/dyn-dep-derivation.json

@@ -7,7 +7,8 @@
   "env": {
     "BIG_BAD": "WOLF"
   },
-  "inputDrvs": {
+  "inputs": {
+    "drvs": {
       "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
         "dynamicOutputs": {
           "cat": {
@@ -29,11 +30,12 @@
         ]
       }
     },
-  "inputSrcs": [
+    "srcs": [
       "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
-  ],
+    ]
+  },
   "name": "dyn-dep-derivation",
   "outputs": {},
   "system": "wasm-sel4",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json

@@ -10,8 +10,10 @@
     "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults",
     "system": "my-system"
   },
-  "inputDrvs": {},
+  "inputs": {
-  "inputSrcs": [],
+    "drvs": {},
+    "srcs": []
+  },
   "name": "advanced-attributes-defaults",
   "outputs": {
     "out": {
@@ -19,5 +21,5 @@
     }
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json

@@ -8,8 +8,10 @@
     "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev",
     "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults"
   },
-  "inputDrvs": {},
+  "inputs": {
-  "inputSrcs": [],
+    "drvs": {},
+    "srcs": []
+  },
   "name": "advanced-attributes-structured-attrs-defaults",
   "outputs": {
     "dev": {
@@ -29,5 +31,5 @@
     "system": "my-system"
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json

@@ -9,7 +9,8 @@
     "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev",
     "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
   },
-  "inputDrvs": {
+  "inputs": {
+    "drvs": {
       "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
         "dynamicOutputs": {},
         "outputs": [
@@ -25,9 +26,10 @@
         ]
       }
     },
-  "inputSrcs": [
+    "srcs": [
       "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
-  ],
+    ]
+  },
   "name": "advanced-attributes-structured-attrs",
   "outputs": {
     "bin": {
@@ -96,5 +98,5 @@
     "system": "my-system"
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/ia/advanced-attributes.json

@@ -23,7 +23,8 @@
     "requiredSystemFeatures": "rainbow uid-range",
     "system": "my-system"
   },
-  "inputDrvs": {
+  "inputs": {
+    "drvs": {
       "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
         "dynamicOutputs": {},
         "outputs": [
@@ -39,9 +40,10 @@
         ]
       }
     },
-  "inputSrcs": [
+    "srcs": [
       "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
-  ],
+    ]
+  },
   "name": "advanced-attributes",
   "outputs": {
     "out": {
@@ -49,5 +51,5 @@
     }
   },
   "system": "my-system",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/data/derivation/output-caFixedFlat.json

@@ -1,5 +1,8 @@
 {
-  "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+  "hash": {
-  "hashAlgo": "sha256",
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="
+  },
   "method": "flat"
 }

src/libstore-tests/data/derivation/output-caFixedNAR.json

@@ -1,5 +1,8 @@
 {
-  "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+  "hash": {
-  "hashAlgo": "sha256",
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="
+  },
   "method": "nar"
 }

src/libstore-tests/data/derivation/output-caFixedText.json

@@ -1,5 +1,8 @@
 {
-  "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
+  "hash": {
-  "hashAlgo": "sha256",
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="
+  },
   "method": "text"
 }

src/libstore-tests/data/derivation/simple-derivation.json

@@ -7,7 +7,8 @@
   "env": {
     "BIG_BAD": "WOLF"
   },
-  "inputDrvs": {
+  "inputs": {
+    "drvs": {
       "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
         "dynamicOutputs": {},
         "outputs": [
@@ -16,11 +17,12 @@
         ]
       }
     },
-  "inputSrcs": [
+    "srcs": [
       "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
-  ],
+    ]
+  },
   "name": "simple-derivation",
   "outputs": {},
   "system": "wasm-sel4",
-  "version": 3
+  "version": 4
 }

src/libstore-tests/nix_api_store.cc

@@ -636,7 +636,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
     auto outj_ph = nix::hashPlaceholder("outj");
 
     std::string drvJson = R"({
-        "version": 3,
+        "version": 4,
         "name": "multi-output-test",
         "system": ")" + nix::settings.thisSystem.get()
                           + R"(",
@@ -668,8 +668,10 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
             "outa": ")" + outa_ph
                           + R"("
         },
-        "inputDrvs": {},
+        "inputs": {
-        "inputSrcs": [],
+          "drvs": {},
+          "srcs": []
+        },
         "outputs": {
             "outd": { "hashAlgo": "sha256", "method": "nar" },
             "outf": { "hashAlgo": "sha256", "method": "nar" },

src/libstore/derivations.cc

@@ -1293,15 +1293,13 @@ void adl_serializer<DerivationOutput>::to_json(json & res, const DerivationOutpu
         overloaded{
             [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; },
             [&](const DerivationOutput::CAFixed & dof) {
+                res = dof.ca;
+        // FIXME print refs?
         /* it would be nice to output the path for user convenience, but
            this would require us to know the store dir. */
 #if 0
                 res["path"] = dof.path(store, drvName, outputName);
 #endif
-                res["method"] = std::string{dof.ca.method.render()};
-                res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo);
-                res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false);
-                // FIXME print refs?
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 res["method"] = std::string{dof.method.render()};
@@ -1341,15 +1339,12 @@ adl_serializer<DerivationOutput>::from_json(const json & _json, const Experiment
         };
     }
 
-    else if (keys == (std::set<std::string_view>{"method", "hashAlgo", "hash"})) {
+    else if (keys == (std::set<std::string_view>{"method", "hash"})) {
-        auto [method, hashAlgo] = methodAlgo();
         auto dof = DerivationOutput::CAFixed{
-            .ca =
+            .ca = static_cast<ContentAddress>(_json),
-                ContentAddress{
-                    .method = std::move(method),
-                    .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo),
-                },
         };
+        if (dof.ca.method == ContentAddressMethod::Raw::Text)
+            xpSettings.require(Xp::DynamicDerivations, "text-hashed derivation output in JSON");
         /* We no longer produce this (denormalized) field (for the
            reasons described above), so we don't need to check it. */
 #if 0
@@ -1392,7 +1387,7 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
 
     res["name"] = d.name;
 
-    res["version"] = 3;
+    res["version"] = 4;
 
     {
         nlohmann::json & outputsObj = res["outputs"];
@@ -1403,13 +1398,16 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
     }
 
     {
-        auto & inputsList = res["inputSrcs"];
+        auto & inputsObj = res["inputs"];
+        inputsObj = nlohmann::json::object();
+
+        {
+            auto & inputsList = inputsObj["srcs"];
             inputsList = nlohmann::json::array();
             for (auto & input : d.inputSrcs)
                 inputsList.emplace_back(input);
         }
 
-    {
         auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json {
             auto value = nlohmann::json::object();
             value["outputs"] = inputNode.value;
@@ -1421,14 +1419,13 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
             }
             return value;
         };
-        {
+
-            auto & inputDrvsObj = res["inputDrvs"];
+        auto & inputDrvsObj = inputsObj["drvs"];
         inputDrvsObj = nlohmann::json::object();
         for (auto & [inputDrv, inputNode] : d.inputDrvs.map) {
             inputDrvsObj[inputDrv.to_string()] = doInput(inputNode);
         }
     }
-    }
 
     res["system"] = d.platform;
     res["builder"] = d.builder;
@@ -1449,8 +1446,8 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
 
     res.name = getString(valueAt(json, "name"));
 
-    if (valueAt(json, "version") != 3)
+    if (valueAt(json, "version") != 4)
-        throw Error("Only derivation format version 3 is currently supported.");
+        throw Error("Only derivation format version 4 is currently supported.");
 
     try {
         auto outputs = getObject(valueAt(json, "outputs"));
@@ -1463,11 +1460,14 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
     }
 
     try {
-        auto inputSrcs = getArray(valueAt(json, "inputSrcs"));
+        auto inputsObj = getObject(valueAt(json, "inputs"));
+
+        try {
+            auto inputSrcs = getArray(valueAt(inputsObj, "srcs"));
             for (auto & input : inputSrcs)
                 res.inputSrcs.insert(input);
         } catch (Error & e) {
-        e.addTrace({}, "while reading key 'inputSrcs'");
+            e.addTrace({}, "while reading key 'srcs'");
             throw;
         }
 
@@ -1484,11 +1484,15 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
                 }
                 return node;
             };
-        auto drvs = getObject(valueAt(json, "inputDrvs"));
+            auto drvs = getObject(valueAt(inputsObj, "drvs"));
             for (auto & [inputDrvPath, inputOutputs] : drvs)
                 res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs);
         } catch (Error & e) {
-        e.addTrace({}, "while reading key 'inputDrvs'");
+            e.addTrace({}, "while reading key 'drvs'");
+            throw;
+        }
+    } catch (Error & e) {
+        e.addTrace({}, "while reading key 'inputs'");
         throw;
     }
 

tests/functional/dyn-drv/non-trivial.nix

@@ -51,10 +51,12 @@ builtins.outputOf
             "$word": "hello, from $word!",
             "PATH": ${builtins.toJSON path}
           },
-          "inputDrvs": {
+          "inputs": {
+            "drvs": {
               $inputDrvs
             },
-          "inputSrcs": [],
+            "srcs": []
+          },
           "name": "build-$word",
           "outputs": {
             "out": {
@@ -63,7 +65,7 @@ builtins.outputOf
             }
           },
           "system": "${system}",
-          "version": 3
+          "version": 4
         }
       EOF
         drvPath=$(echo "$json" | nix derivation add)