From 0c37a62207f52aa9d58b8d18a4860dec3a270f72 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 00:24:12 -0400 Subject: [PATCH 1/3] Change JSON derivation format in two ways - Use canonical content address JSON format for floating content addressed derivation outputs This keeps it more consistent. - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) This will allow for an easier to use, but less compact, alternative where `srcs` is just a list of derived paths. It also allows for other experiments for derivations with a different input structure, as I suspect will be needed for secure build traces. --- .../source/protocols/json/derivation.md | 4 +- doc/manual/source/protocols/json/meson.build | 2 +- ...{derivation-v3.yaml => derivation-v4.yaml} | 114 ++++++++++-------- src/json-schema-checks/meson.build | 14 +-- .../ca/advanced-attributes-defaults.json | 8 +- ...-attributes-structured-attrs-defaults.json | 8 +- .../advanced-attributes-structured-attrs.json | 38 +++--- .../derivation/ca/advanced-attributes.json | 38 +++--- .../data/derivation/ca/self-contained.json | 8 +- .../data/derivation/dyn-dep-derivation.json | 50 ++++---- .../ia/advanced-attributes-defaults.json | 8 +- ...-attributes-structured-attrs-defaults.json | 8 +- .../advanced-attributes-structured-attrs.json | 38 +++--- .../derivation/ia/advanced-attributes.json | 38 +++--- .../data/derivation/output-caFixedFlat.json | 7 +- .../data/derivation/output-caFixedNAR.json | 7 +- .../data/derivation/output-caFixedText.json | 7 +- .../data/derivation/simple-derivation.json | 26 ++-- src/libstore-tests/nix_api_store.cc | 8 +- src/libstore/derivations.cc | 108 +++++++++-------- tests/functional/dyn-drv/non-trivial.nix | 10 +- 21 files changed, 298 insertions(+), 251 deletions(-) rename doc/manual/source/protocols/json/schema/{derivation-v3.yaml => derivation-v4.yaml} (78%) diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index a4a4ea79d..6eafb255e 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,7 +1,7 @@ -{{#include derivation-v3-fixed.md}} +{{#include derivation-v4-fixed.md}} diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c56de49c7..c0b8416d7 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -13,7 +13,7 @@ schemas = [ 'content-address-v1', 'store-path-v1', 'store-object-info-v1', - 'derivation-v3', + 'derivation-v4', 'deriving-path-v1', 'build-trace-entry-v1', 'build-result-v1', diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml similarity index 78% rename from doc/manual/source/protocols/json/schema/derivation-v3.yaml rename to doc/manual/source/protocols/json/schema/derivation-v4.yaml index fa68adcb1..2528f7502 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -1,8 +1,8 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v4.json" title: Derivation description: | - Experimental JSON representation of a Nix derivation (version 3). + Experimental JSON representation of a Nix derivation (version 4). This schema describes the JSON representation of Nix's `Derivation` type. @@ -17,8 +17,7 @@ required: - name - version - outputs - - inputSrcs - - inputDrvs + - inputs - system - builder - args @@ -32,10 +31,10 @@ properties: Used when calculating store paths for the derivation’s outputs. version: - const: 3 - title: Format version (must be 3) + const: 4 + title: Format version (must be 4) description: | - Must be `3`. + Must be `4`. This is a guard that allows us to continue evolving this format. The choice of `3` is fairly arbitrary, but corresponds to this informal version: @@ -47,6 +46,12 @@ properties: - Version 3: Drop store dir from store paths, just include base name. + - Version 4: Two cleanups, batched together to lesson churn: + + - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) + + - Use canonical content address JSON format for floating content addressed derivation outputs. + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. outputs: @@ -70,47 +75,56 @@ properties: additionalProperties: "$ref": "#/$defs/output/overall" - inputSrcs: - type: array - title: Input source paths - description: | - List of store paths on which this derivation depends. - - > **Example** - > - > ```json - > "inputSrcs": [ - > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", - > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" - > ] - > ``` - items: - $ref: "store-path-v1.yaml" - - inputDrvs: + inputs: type: object - title: Input derivations + title: Derivation inputs description: | - Mapping of derivation paths to lists of output names they provide. - - > **Example** - > - > ```json - > "inputDrvs": { - > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] - > } - > ``` - > - > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. - patternProperties: - "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": - title: Store Path + Input dependencies for the derivation, organized into source paths and derivation dependencies. + required: + - srcs + - drvs + properties: + srcs: + type: array + title: Input source paths description: | - A store path to a derivation, mapped to the outputs of that derivation. - oneOf: - - "$ref": "#/$defs/outputNames" - - "$ref": "#/$defs/dynamicOutputs" + List of store paths on which this derivation depends. + + > **Example** + > + > ```json + > "srcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + items: + $ref: "store-path-v1.yaml" + drvs: + type: object + title: Input derivations + description: | + Mapping of derivation paths to lists of output names they provide. + + > **Example** + > + > ```json + > "drvs": { + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > } + > ``` + > + > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + patternProperties: + "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": + title: Store Path + description: | + A store path to a derivation, mapped to the outputs of that derivation. + oneOf: + - "$ref": "#/$defs/outputNames" + - "$ref": "#/$defs/dynamicOutputs" + additionalProperties: false additionalProperties: false system: @@ -189,24 +203,18 @@ properties: The output is content-addressed, and the content-address is fixed in advance. See [Fixed-output content-addressing](@docroot@/store/derivation/outputs/content-address.md#fixed) for more details. - type: object + "$ref": "./content-address-v1.yaml" required: - method - - hashAlgo - hash properties: method: - "$ref": "./content-address-v1.yaml#/$defs/method" description: | Method of content addressing used for this output. - hashAlgo: - title: Hash algorithm - "$ref": "./hash-v1.yaml#/$defs/algorithm" hash: - type: string title: Expected hash value description: | - The expected content hash in base-16. + The expected content hash. additionalProperties: false caFloating: diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 65a2651b7..fedacedeb 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -70,7 +70,7 @@ schemas += [ # Match overall { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml', + 'schema' : schema_dir / 'derivation-v4.yaml', 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', @@ -78,7 +78,7 @@ schemas += [ }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/overall', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/overall', 'files' : [ 'output-caFixedFlat.json', 'output-caFixedNAR.json', @@ -92,14 +92,14 @@ schemas += [ # Match exact variant { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/inputAddressed', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/inputAddressed', 'files' : [ 'output-inputAddressed.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFixed', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFixed', 'files' : [ 'output-caFixedFlat.json', 'output-caFixedNAR.json', @@ -108,21 +108,21 @@ schemas += [ }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFloating', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFloating', 'files' : [ 'output-caFloating.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/deferred', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/deferred', 'files' : [ 'output-deferred.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/impure', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/impure', 'files' : [ 'output-impure.json', ], diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json index eb4bd4f3d..781b4cb14 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -12,8 +12,10 @@ "outputHashMode": "recursive", "system": "my-system" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-defaults", "outputs": { "out": { @@ -22,5 +24,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 3a4a3079b..7437b51ef 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -8,8 +8,10 @@ "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { @@ -33,5 +35,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index b10355af7..2a4e70558 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -9,25 +9,27 @@ "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, - "inputDrvs": { - "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] }, - "inputSrcs": [ - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" - ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { @@ -101,5 +103,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index d66882036..55dbe62e0 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -25,25 +25,27 @@ "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" }, - "inputDrvs": { - "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] }, - "inputSrcs": [ - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" - ], "name": "advanced-attributes", "outputs": { "out": { @@ -52,5 +54,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/self-contained.json b/src/libstore-tests/data/derivation/ca/self-contained.json index 331beb7be..c05710140 100644 --- a/src/libstore-tests/data/derivation/ca/self-contained.json +++ b/src/libstore-tests/data/derivation/ca/self-contained.json @@ -10,8 +10,10 @@ "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", "system": "x86_64-linux" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "myname", "outputs": { "out": { @@ -20,5 +22,5 @@ } }, "system": "x86_64-linux", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/dyn-dep-derivation.json b/src/libstore-tests/data/derivation/dyn-dep-derivation.json index 1a9f54c53..1793c5f2d 100644 --- a/src/libstore-tests/data/derivation/dyn-dep-derivation.json +++ b/src/libstore-tests/data/derivation/dyn-dep-derivation.json @@ -7,33 +7,35 @@ "env": { "BIG_BAD": "WOLF" }, - "inputDrvs": { - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": [ - "kitten" - ] + "inputs": { + "drvs": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } }, - "goose": { - "dynamicOutputs": {}, - "outputs": [ - "gosling" - ] - } - }, - "outputs": [ - "cat", - "dog" - ] - } + "outputs": [ + "cat", + "dog" + ] + } + }, + "srcs": [ + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ] }, - "inputSrcs": [ - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], "name": "dyn-dep-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json index 0fa543f21..898762123 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json @@ -10,8 +10,10 @@ "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", "system": "my-system" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-defaults", "outputs": { "out": { @@ -19,5 +21,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index e02392ea1..c51095986 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -8,8 +8,10 @@ "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { @@ -29,5 +31,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index 9230b06b6..e07d1294b 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -9,25 +9,27 @@ "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev", "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, - "inputDrvs": { - "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] }, - "inputSrcs": [ - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" - ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { @@ -96,5 +98,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index ba5911c91..372b4fbb9 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -23,25 +23,27 @@ "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" }, - "inputDrvs": { - "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] }, - "inputSrcs": [ - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" - ], "name": "advanced-attributes", "outputs": { "out": { @@ -49,5 +51,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json index e6a0123f6..9a38608b3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedFlat.json +++ b/src/libstore-tests/data/derivation/output-caFixedFlat.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "flat" } diff --git a/src/libstore-tests/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json index b57e065a9..767c605a3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedNAR.json +++ b/src/libstore-tests/data/derivation/output-caFixedNAR.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "nar" } diff --git a/src/libstore-tests/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json index 84778509e..a04f1ff2a 100644 --- a/src/libstore-tests/data/derivation/output-caFixedText.json +++ b/src/libstore-tests/data/derivation/output-caFixedText.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "text" } diff --git a/src/libstore-tests/data/derivation/simple-derivation.json b/src/libstore-tests/data/derivation/simple-derivation.json index 41a049aef..04129a096 100644 --- a/src/libstore-tests/data/derivation/simple-derivation.json +++ b/src/libstore-tests/data/derivation/simple-derivation.json @@ -7,20 +7,22 @@ "env": { "BIG_BAD": "WOLF" }, - "inputDrvs": { - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } + "inputs": { + "drvs": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "srcs": [ + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ] }, - "inputSrcs": [ - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], "name": "simple-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 228b8069f..bf411053a 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -636,7 +636,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) auto outj_ph = nix::hashPlaceholder("outj"); std::string drvJson = R"({ - "version": 3, + "version": 4, "name": "multi-output-test", "system": ")" + nix::settings.thisSystem.get() + R"(", @@ -668,8 +668,10 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) "outa": ")" + outa_ph + R"(" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "outputs": { "outd": { "hashAlgo": "sha256", "method": "nar" }, "outf": { "hashAlgo": "sha256", "method": "nar" }, diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index e6ac08fd9..31ca167f9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1293,15 +1293,13 @@ void adl_serializer::to_json(json & res, const DerivationOutpu overloaded{ [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, [&](const DerivationOutput::CAFixed & dof) { - /* it would be nice to output the path for user convenience, but - this would require us to know the store dir. */ + res = dof.ca; + // FIXME print refs? + /* it would be nice to output the path for user convenience, but + this would require us to know the store dir. */ #if 0 res["path"] = dof.path(store, drvName, outputName); #endif - res["method"] = std::string{dof.ca.method.render()}; - res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); - res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); - // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { res["method"] = std::string{dof.method.render()}; @@ -1341,15 +1339,12 @@ adl_serializer::from_json(const json & _json, const Experiment }; } - else if (keys == (std::set{"method", "hashAlgo", "hash"})) { - auto [method, hashAlgo] = methodAlgo(); + else if (keys == (std::set{"method", "hash"})) { auto dof = DerivationOutput::CAFixed{ - .ca = - ContentAddress{ - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), - }, + .ca = static_cast(_json), }; + if (dof.ca.method == ContentAddressMethod::Raw::Text) + xpSettings.require(Xp::DynamicDerivations, "text-hashed derivation output in JSON"); /* We no longer produce this (denormalized) field (for the reasons described above), so we don't need to check it. */ #if 0 @@ -1392,7 +1387,7 @@ void adl_serializer::to_json(json & res, const Derivation & d) res["name"] = d.name; - res["version"] = 3; + res["version"] = 4; { nlohmann::json & outputsObj = res["outputs"]; @@ -1403,13 +1398,16 @@ void adl_serializer::to_json(json & res, const Derivation & d) } { - auto & inputsList = res["inputSrcs"]; - inputsList = nlohmann::json::array(); - for (auto & input : d.inputSrcs) - inputsList.emplace_back(input); - } + auto & inputsObj = res["inputs"]; + inputsObj = nlohmann::json::object(); + + { + auto & inputsList = inputsObj["srcs"]; + inputsList = nlohmann::json::array(); + for (auto & input : d.inputSrcs) + inputsList.emplace_back(input); + } - { auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { auto value = nlohmann::json::object(); value["outputs"] = inputNode.value; @@ -1421,12 +1419,11 @@ void adl_serializer::to_json(json & res, const Derivation & d) } return value; }; - { - auto & inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { - inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } + + auto & inputDrvsObj = inputsObj["drvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); } } @@ -1449,8 +1446,8 @@ Derivation adl_serializer::from_json(const json & _json, const Exper res.name = getString(valueAt(json, "name")); - if (valueAt(json, "version") != 3) - throw Error("Only derivation format version 3 is currently supported."); + if (valueAt(json, "version") != 4) + throw Error("Only derivation format version 4 is currently supported."); try { auto outputs = getObject(valueAt(json, "outputs")); @@ -1463,32 +1460,39 @@ Derivation adl_serializer::from_json(const json & _json, const Exper } try { - auto inputSrcs = getArray(valueAt(json, "inputSrcs")); - for (auto & input : inputSrcs) - res.inputSrcs.insert(input); - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputSrcs'"); - throw; - } + auto inputsObj = getObject(valueAt(json, "inputs")); - try { - auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { - auto & json = getObject(_json); - DerivedPathMap::ChildNode node; - node.value = getStringSet(valueAt(json, "outputs")); - auto drvs = getObject(valueAt(json, "dynamicOutputs")); - for (auto & [outputId, childNode] : drvs) { - xpSettings.require( - Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); - node.childMap[outputId] = doInput(childNode); - } - return node; - }; - auto drvs = getObject(valueAt(json, "inputDrvs")); - for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); + try { + auto inputSrcs = getArray(valueAt(inputsObj, "srcs")); + for (auto & input : inputSrcs) + res.inputSrcs.insert(input); + } catch (Error & e) { + e.addTrace({}, "while reading key 'srcs'"); + throw; + } + + try { + auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { + auto & json = getObject(_json); + DerivedPathMap::ChildNode node; + node.value = getStringSet(valueAt(json, "outputs")); + auto drvs = getObject(valueAt(json, "dynamicOutputs")); + for (auto & [outputId, childNode] : drvs) { + xpSettings.require( + Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + auto drvs = getObject(valueAt(inputsObj, "drvs")); + for (auto & [inputDrvPath, inputOutputs] : drvs) + res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'drvs'"); + throw; + } } catch (Error & e) { - e.addTrace({}, "while reading key 'inputDrvs'"); + e.addTrace({}, "while reading key 'inputs'"); throw; } diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix index 3c24ac2ee..87f2d9cfe 100644 --- a/tests/functional/dyn-drv/non-trivial.nix +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -51,10 +51,12 @@ builtins.outputOf "$word": "hello, from $word!", "PATH": ${builtins.toJSON path} }, - "inputDrvs": { - $inputDrvs + "inputs": { + "drvs": { + $inputDrvs + }, + "srcs": [] }, - "inputSrcs": [], "name": "build-$word", "outputs": { "out": { @@ -63,7 +65,7 @@ builtins.outputOf } }, "system": "${system}", - "version": 3 + "version": 4 } EOF drvPath=$(echo "$json" | nix derivation add) From caa196e31d00df3fa31d8fc47ae2efb8eb5ac6d4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 17:44:07 -0400 Subject: [PATCH 2/3] Make the store path info `ca` field structured in JSON The old string format is a holdover from the pre JSON days. It is not friendly to users who need to get the information out of it. Also introduce the sort of versioning we have for derivation for this format too. --- doc/manual/source/protocols/json/meson.build | 2 +- ...re-object-info-v1 => store-object-info-v2} | 0 ...info-v1.yaml => store-object-info-v2.yaml} | 31 ++++++++++++++++--- .../protocols/json/store-object-info.md | 12 +++---- src/json-schema-checks/meson.build | 12 +++---- src/libstore-tests/data/nar-info/impure.json | 12 +++++-- src/libstore-tests/data/nar-info/pure.json | 12 +++++-- .../data/path-info/empty_impure.json | 3 +- .../data/path-info/empty_pure.json | 3 +- src/libstore-tests/data/path-info/impure.json | 12 +++++-- src/libstore-tests/data/path-info/pure.json | 12 +++++-- src/libstore/path-info.cc | 24 ++++++++++++-- tests/functional/fixed.sh | 11 ++++++- tests/functional/git-hashing/simple-common.sh | 14 +++++++-- tests/functional/impure-derivations.sh | 2 +- tests/functional/nix-profile.sh | 2 +- tests/functional/signing.sh | 2 +- 17 files changed, 130 insertions(+), 36 deletions(-) rename doc/manual/source/protocols/json/schema/{store-object-info-v1 => store-object-info-v2} (100%) rename doc/manual/source/protocols/json/schema/{store-object-info-v1.yaml => store-object-info-v2.yaml} (91%) diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c0b8416d7..4ab94c63b 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -12,7 +12,7 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v1', + 'store-object-info-v2', 'derivation-v4', 'deriving-path-v1', 'build-trace-entry-v1', diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1 b/doc/manual/source/protocols/json/schema/store-object-info-v2 similarity index 100% rename from doc/manual/source/protocols/json/schema/store-object-info-v1 rename to doc/manual/source/protocols/json/schema/store-object-info-v2 diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml similarity index 91% rename from doc/manual/source/protocols/json/schema/store-object-info-v1.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v2.yaml index d79f25043..4f442e0c3 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml @@ -1,6 +1,6 @@ -"$schema": "http://json-schema.org/draft-07/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v1.json" -title: Store Object Info +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" +title: Store Object Info v2 description: | Information about a [store object](@docroot@/store/store-object.md). @@ -41,11 +41,27 @@ $defs: This is the minimal set of fields that describe what a store object contains. type: object required: + - version - narHash - narSize - references - ca properties: + version: + type: integer + const: 2 + title: Format version (must be 2) + description: | + Must be `2`. + This is a guard that allows us to continue evolving this format. + Here is the rough version history: + + - Version 0: `.narinfo` line-oriented format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from `.narinfo` format. + + - Version 2: Use structured JSON type for `ca` + path: type: string title: Store Path @@ -76,7 +92,10 @@ $defs: type: string ca: - type: ["string", "null"] + oneOf: + - type: "null" + const: null + - "$ref": "./content-address-v1.yaml" title: Content Address description: | If the store object is [content-addressed](@docroot@/store/store-object/content-address.md), @@ -91,6 +110,7 @@ $defs: In other words, the same store object in different stores could have different values for these impure fields. type: object required: + - version - narHash - narSize - references @@ -101,6 +121,7 @@ $defs: - ultimate - signatures properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } @@ -164,6 +185,7 @@ $defs: This download information, being specific to how the store object happens to be stored and transferred, is also considered to be non-intrinsic / impure. type: object required: + - version - narHash - narSize - references @@ -179,6 +201,7 @@ $defs: - downloadHash - downloadSize properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4673dd773..6a101ab0f 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,29 +1,29 @@ -{{#include store-object-info-v1-fixed.md}} +{{#include store-object-info-v2-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v1/pure.json}} +{{#include schema/store-object-info-v2/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v1/impure.json}} +{{#include schema/store-object-info-v2/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v1/empty_pure.json}} +{{#include schema/store-object-info-v2/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v1/empty_impure.json}} +{{#include schema/store-object-info-v2/empty_impure.json}} ``` ### NAR info (minimal) @@ -41,5 +41,5 @@ diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index fedacedeb..f72affb0b 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -134,7 +134,7 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -144,7 +144,7 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -162,7 +162,7 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', 'empty_pure.json', @@ -170,7 +170,7 @@ schemas += [ }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', 'files' : [ 'impure.json', 'empty_impure.json', @@ -178,14 +178,14 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', 'files' : [ 'impure.json', ], diff --git a/src/libstore-tests/data/nar-info/impure.json b/src/libstore-tests/data/nar-info/impure.json index bb9791a6a..f35ff990b 100644 --- a/src/libstore-tests/data/nar-info/impure.json +++ b/src/libstore-tests/data/nar-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "compression": "xz", "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", @@ -16,5 +23,6 @@ "qwer" ], "ultimate": true, - "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 2 } diff --git a/src/libstore-tests/data/nar-info/pure.json b/src/libstore-tests/data/nar-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/nar-info/pure.json +++ b/src/libstore-tests/data/nar-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_impure.json b/src/libstore-tests/data/path-info/empty_impure.json index be982dcef..381acaa03 100644 --- a/src/libstore-tests/data/path-info/empty_impure.json +++ b/src/libstore-tests/data/path-info/empty_impure.json @@ -6,5 +6,6 @@ "references": [], "registrationTime": null, "signatures": [], - "ultimate": false + "ultimate": false, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_pure.json b/src/libstore-tests/data/path-info/empty_pure.json index 10d9f508a..6d3fa646b 100644 --- a/src/libstore-tests/data/path-info/empty_pure.json +++ b/src/libstore-tests/data/path-info/empty_pure.json @@ -2,5 +2,6 @@ "ca": null, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 0, - "references": [] + "references": [], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/impure.json b/src/libstore-tests/data/path-info/impure.json index 0c452cc49..141b38a16 100644 --- a/src/libstore-tests/data/path-info/impure.json +++ b/src/libstore-tests/data/path-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, @@ -12,5 +19,6 @@ "asdf", "qwer" ], - "ultimate": true + "ultimate": true, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/pure.json b/src/libstore-tests/data/path-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/path-info/pure.json +++ b/src/libstore-tests/data/path-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 09a78a4ad..c535d08f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -156,6 +156,8 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf auto jsonObject = json::object(); + jsonObject["version"] = 2; + jsonObject["narHash"] = narHash.to_string(hashFormat, true); jsonObject["narSize"] = narSize; @@ -165,7 +167,7 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf jsonRefs.emplace_back(store.printStorePath(ref)); } - jsonObject["ca"] = ca ? (std::optional{renderContentAddress(*ca)}) : std::nullopt; + jsonObject["ca"] = ca; if (includeImpureInfo) { jsonObject["deriver"] = deriver ? (std::optional{store.printStorePath(*deriver)}) : std::nullopt; @@ -189,6 +191,16 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store }; auto & json = getObject(_json); + + // Check version (optional for backward compatibility) + nlohmann::json::number_unsigned_t version = 1; + if (json.contains("version")) { + version = getUnsigned(valueAt(json, "version")); + if (version != 1 && version != 2) { + throw Error("Unsupported path info JSON format version %d, expected 1 through 2", version); + } + } + res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); res.narSize = getUnsigned(valueAt(json, "narSize")); @@ -205,7 +217,15 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store // missing is for back-compat. if (auto * rawCa0 = optionalValueAt(json, "ca")) if (auto * rawCa = getNullable(*rawCa0)) - res.ca = ContentAddress::parse(getString(*rawCa)); + switch (version) { + case 1: + // old string format also used in SQLite DB and .narinfo + res.ca = ContentAddress::parse(getString(*rawCa)); + break; + case 2 ... std::numeric_limits::max(): + res.ca = *rawCa; + break; + } if (auto * rawDeriver0 = optionalValueAt(json, "deriver")) if (auto * rawDeriver = getNullable(*rawDeriver0)) diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index edf6f88d4..7861392ec 100755 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -14,7 +14,16 @@ nix-build fixed.nix -A bad --no-out-link && fail "should fail" # Building with the bad hash should produce the "good" output path as # a side-effect. [[ -e $path ]] -nix path-info --json "$path" | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd +nix path-info --json "$path" | jq -e \ + --arg hash "$(nix hash convert --to base64 "md5:8ddd8be4b179a529afa5f2ffae4b9858")" \ + '.[].ca == { + method: "flat", + hash: { + algorithm: "md5", + format: "base64", + hash: $hash + }, + }' echo 'testing good...' nix-build fixed.nix -A good --no-out-link diff --git a/tests/functional/git-hashing/simple-common.sh b/tests/functional/git-hashing/simple-common.sh index 08b5c0e71..eaa0a9529 100644 --- a/tests/functional/git-hashing/simple-common.sh +++ b/tests/functional/git-hashing/simple-common.sh @@ -47,9 +47,17 @@ try2 () { hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath") [[ "$hashFromGit" == "$expected" ]] - local caFromNix - caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca") - [[ "fixed:git:$hashAlgo:$(nix hash convert --to nix32 "$hashAlgo:$hashFromGit")" = "$caFromNix" ]] + nix path-info --json "$path" | jq -e \ + --arg algo "$hashAlgo" \ + --arg hash "$(nix hash convert --to base64 "$hashAlgo:$hashFromGit")" \ + '.[].ca == { + method: "git", + hash: { + algorithm: $algo, + format: "base64", + hash: $hash + }, + }' } test0 () { diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index e0b7c3eea..211abccb0 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -30,7 +30,7 @@ path1_stuff=$(echo "$json" | jq -r .[].outputs.stuff) [[ $(< "$path1"/n) = 0 ]] [[ $(< "$path1_stuff"/bla) = 0 ]] -[[ $(nix path-info --json "$path1" | jq .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$path1" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out) [[ $(< "$path2"/n) = 1 ]] diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 922162d4b..494b24ddb 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -166,7 +166,7 @@ printf 4.0 > "$flake1Dir"/version printf Utrecht > "$flake1Dir"/who nix profile add "$flake1Dir" [[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[[ $(nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -r .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' # Override the outputs. nix profile remove simple flake1 diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 2893efec7..1bcaf2f53 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -58,7 +58,7 @@ nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' +nix path-info --json "$outPathCA" | jq -e '.[].ca | .method == "flat" and .hash.algorithm == "md5"' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. From 8cc3ede0fac902c66a9563a9924c54eda66d48c9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 31 Oct 2025 00:43:45 -0400 Subject: [PATCH 3/3] Add change-log entry for derivation format changes --- doc/manual/rl-next/json-format-changes.md | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 doc/manual/rl-next/json-format-changes.md diff --git a/doc/manual/rl-next/json-format-changes.md b/doc/manual/rl-next/json-format-changes.md new file mode 100644 index 000000000..c5518ee1b --- /dev/null +++ b/doc/manual/rl-next/json-format-changes.md @@ -0,0 +1,47 @@ +--- +synopsis: "JSON format changes for store path info and derivations" +prs: [] +issues: [] +--- + +JSON formats for store path info and derivations have been updated with new versions and structured fields. + +## Store Path Info JSON (Version 2) + +The store path info JSON format has been updated from version 1 to version 2: + +- **Added `version` field**: + + All store path info JSON now includes `"version": 2`. + +- **Structured `ca` field**: + + Content address is now a structured JSON object instead of a string: + + - Old: `"ca": "fixed:r:sha256:1abc..."` + - New: `"ca": {"method": "nar", "hash": {"algorithm": "sha256", "format": "base64", "hash": "EMIJ+giQ..."}}` + - Still `null` values for input-addressed store objects + +Version 1 format is still accepted when reading for backward compatibility. + +**Affected command**: `nix path-info --json` + +## Derivation JSON (Version 4) + +The derivation JSON format has been updated from version 3 to version 4: + +- **Restructured inputs**: + + Inputs are now nested under an `inputs` object: + + - Old: `"inputSrcs": [...], "inputDrvs": {...}` + - New: `"inputs": {"srcs": [...], "drvs": {...}}` + +- **Consistent content addresses**: + + Floating content-addressed outputs now use structured JSON format. + This is the same format as `ca` in in store path info (after the new version). + +Version 3 and earlier formats are *not* accepted when reading. + +**Affected command**: `nix derivation`, namely it's `show` and `add` sub-commands.