From caa196e31d00df3fa31d8fc47ae2efb8eb5ac6d4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 17:44:07 -0400 Subject: [PATCH] Make the store path info `ca` field structured in JSON The old string format is a holdover from the pre JSON days. It is not friendly to users who need to get the information out of it. Also introduce the sort of versioning we have for derivation for this format too. --- doc/manual/source/protocols/json/meson.build | 2 +- ...re-object-info-v1 => store-object-info-v2} | 0 ...info-v1.yaml => store-object-info-v2.yaml} | 31 ++++++++++++++++--- .../protocols/json/store-object-info.md | 12 +++---- src/json-schema-checks/meson.build | 12 +++---- src/libstore-tests/data/nar-info/impure.json | 12 +++++-- src/libstore-tests/data/nar-info/pure.json | 12 +++++-- .../data/path-info/empty_impure.json | 3 +- .../data/path-info/empty_pure.json | 3 +- src/libstore-tests/data/path-info/impure.json | 12 +++++-- src/libstore-tests/data/path-info/pure.json | 12 +++++-- src/libstore/path-info.cc | 24 ++++++++++++-- tests/functional/fixed.sh | 11 ++++++- tests/functional/git-hashing/simple-common.sh | 14 +++++++-- tests/functional/impure-derivations.sh | 2 +- tests/functional/nix-profile.sh | 2 +- tests/functional/signing.sh | 2 +- 17 files changed, 130 insertions(+), 36 deletions(-) rename doc/manual/source/protocols/json/schema/{store-object-info-v1 => store-object-info-v2} (100%) rename doc/manual/source/protocols/json/schema/{store-object-info-v1.yaml => store-object-info-v2.yaml} (91%) diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c0b8416d7..4ab94c63b 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -12,7 +12,7 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v1', + 'store-object-info-v2', 'derivation-v4', 'deriving-path-v1', 'build-trace-entry-v1', diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1 b/doc/manual/source/protocols/json/schema/store-object-info-v2 similarity index 100% rename from doc/manual/source/protocols/json/schema/store-object-info-v1 rename to doc/manual/source/protocols/json/schema/store-object-info-v2 diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml similarity index 91% rename from doc/manual/source/protocols/json/schema/store-object-info-v1.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v2.yaml index d79f25043..4f442e0c3 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml @@ -1,6 +1,6 @@ -"$schema": "http://json-schema.org/draft-07/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v1.json" -title: Store Object Info +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" +title: Store Object Info v2 description: | Information about a [store object](@docroot@/store/store-object.md). @@ -41,11 +41,27 @@ $defs: This is the minimal set of fields that describe what a store object contains. type: object required: + - version - narHash - narSize - references - ca properties: + version: + type: integer + const: 2 + title: Format version (must be 2) + description: | + Must be `2`. + This is a guard that allows us to continue evolving this format. + Here is the rough version history: + + - Version 0: `.narinfo` line-oriented format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from `.narinfo` format. + + - Version 2: Use structured JSON type for `ca` + path: type: string title: Store Path @@ -76,7 +92,10 @@ $defs: type: string ca: - type: ["string", "null"] + oneOf: + - type: "null" + const: null + - "$ref": "./content-address-v1.yaml" title: Content Address description: | If the store object is [content-addressed](@docroot@/store/store-object/content-address.md), @@ -91,6 +110,7 @@ $defs: In other words, the same store object in different stores could have different values for these impure fields. type: object required: + - version - narHash - narSize - references @@ -101,6 +121,7 @@ $defs: - ultimate - signatures properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } @@ -164,6 +185,7 @@ $defs: This download information, being specific to how the store object happens to be stored and transferred, is also considered to be non-intrinsic / impure. type: object required: + - version - narHash - narSize - references @@ -179,6 +201,7 @@ $defs: - downloadHash - downloadSize properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4673dd773..6a101ab0f 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,29 +1,29 @@ -{{#include store-object-info-v1-fixed.md}} +{{#include store-object-info-v2-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v1/pure.json}} +{{#include schema/store-object-info-v2/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v1/impure.json}} +{{#include schema/store-object-info-v2/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v1/empty_pure.json}} +{{#include schema/store-object-info-v2/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v1/empty_impure.json}} +{{#include schema/store-object-info-v2/empty_impure.json}} ``` ### NAR info (minimal) @@ -41,5 +41,5 @@ diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index fedacedeb..f72affb0b 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -134,7 +134,7 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -144,7 +144,7 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -162,7 +162,7 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', 'empty_pure.json', @@ -170,7 +170,7 @@ schemas += [ }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', 'files' : [ 'impure.json', 'empty_impure.json', @@ -178,14 +178,14 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', 'files' : [ 'impure.json', ], diff --git a/src/libstore-tests/data/nar-info/impure.json b/src/libstore-tests/data/nar-info/impure.json index bb9791a6a..f35ff990b 100644 --- a/src/libstore-tests/data/nar-info/impure.json +++ b/src/libstore-tests/data/nar-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "compression": "xz", "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", @@ -16,5 +23,6 @@ "qwer" ], "ultimate": true, - "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 2 } diff --git a/src/libstore-tests/data/nar-info/pure.json b/src/libstore-tests/data/nar-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/nar-info/pure.json +++ b/src/libstore-tests/data/nar-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_impure.json b/src/libstore-tests/data/path-info/empty_impure.json index be982dcef..381acaa03 100644 --- a/src/libstore-tests/data/path-info/empty_impure.json +++ b/src/libstore-tests/data/path-info/empty_impure.json @@ -6,5 +6,6 @@ "references": [], "registrationTime": null, "signatures": [], - "ultimate": false + "ultimate": false, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_pure.json b/src/libstore-tests/data/path-info/empty_pure.json index 10d9f508a..6d3fa646b 100644 --- a/src/libstore-tests/data/path-info/empty_pure.json +++ b/src/libstore-tests/data/path-info/empty_pure.json @@ -2,5 +2,6 @@ "ca": null, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 0, - "references": [] + "references": [], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/impure.json b/src/libstore-tests/data/path-info/impure.json index 0c452cc49..141b38a16 100644 --- a/src/libstore-tests/data/path-info/impure.json +++ b/src/libstore-tests/data/path-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, @@ -12,5 +19,6 @@ "asdf", "qwer" ], - "ultimate": true + "ultimate": true, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/pure.json b/src/libstore-tests/data/path-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/path-info/pure.json +++ b/src/libstore-tests/data/path-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 09a78a4ad..c535d08f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -156,6 +156,8 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf auto jsonObject = json::object(); + jsonObject["version"] = 2; + jsonObject["narHash"] = narHash.to_string(hashFormat, true); jsonObject["narSize"] = narSize; @@ -165,7 +167,7 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf jsonRefs.emplace_back(store.printStorePath(ref)); } - jsonObject["ca"] = ca ? (std::optional{renderContentAddress(*ca)}) : std::nullopt; + jsonObject["ca"] = ca; if (includeImpureInfo) { jsonObject["deriver"] = deriver ? (std::optional{store.printStorePath(*deriver)}) : std::nullopt; @@ -189,6 +191,16 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store }; auto & json = getObject(_json); + + // Check version (optional for backward compatibility) + nlohmann::json::number_unsigned_t version = 1; + if (json.contains("version")) { + version = getUnsigned(valueAt(json, "version")); + if (version != 1 && version != 2) { + throw Error("Unsupported path info JSON format version %d, expected 1 through 2", version); + } + } + res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); res.narSize = getUnsigned(valueAt(json, "narSize")); @@ -205,7 +217,15 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store // missing is for back-compat. if (auto * rawCa0 = optionalValueAt(json, "ca")) if (auto * rawCa = getNullable(*rawCa0)) - res.ca = ContentAddress::parse(getString(*rawCa)); + switch (version) { + case 1: + // old string format also used in SQLite DB and .narinfo + res.ca = ContentAddress::parse(getString(*rawCa)); + break; + case 2 ... std::numeric_limits::max(): + res.ca = *rawCa; + break; + } if (auto * rawDeriver0 = optionalValueAt(json, "deriver")) if (auto * rawDeriver = getNullable(*rawDeriver0)) diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index edf6f88d4..7861392ec 100755 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -14,7 +14,16 @@ nix-build fixed.nix -A bad --no-out-link && fail "should fail" # Building with the bad hash should produce the "good" output path as # a side-effect. [[ -e $path ]] -nix path-info --json "$path" | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd +nix path-info --json "$path" | jq -e \ + --arg hash "$(nix hash convert --to base64 "md5:8ddd8be4b179a529afa5f2ffae4b9858")" \ + '.[].ca == { + method: "flat", + hash: { + algorithm: "md5", + format: "base64", + hash: $hash + }, + }' echo 'testing good...' nix-build fixed.nix -A good --no-out-link diff --git a/tests/functional/git-hashing/simple-common.sh b/tests/functional/git-hashing/simple-common.sh index 08b5c0e71..eaa0a9529 100644 --- a/tests/functional/git-hashing/simple-common.sh +++ b/tests/functional/git-hashing/simple-common.sh @@ -47,9 +47,17 @@ try2 () { hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath") [[ "$hashFromGit" == "$expected" ]] - local caFromNix - caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca") - [[ "fixed:git:$hashAlgo:$(nix hash convert --to nix32 "$hashAlgo:$hashFromGit")" = "$caFromNix" ]] + nix path-info --json "$path" | jq -e \ + --arg algo "$hashAlgo" \ + --arg hash "$(nix hash convert --to base64 "$hashAlgo:$hashFromGit")" \ + '.[].ca == { + method: "git", + hash: { + algorithm: $algo, + format: "base64", + hash: $hash + }, + }' } test0 () { diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index e0b7c3eea..211abccb0 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -30,7 +30,7 @@ path1_stuff=$(echo "$json" | jq -r .[].outputs.stuff) [[ $(< "$path1"/n) = 0 ]] [[ $(< "$path1_stuff"/bla) = 0 ]] -[[ $(nix path-info --json "$path1" | jq .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$path1" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out) [[ $(< "$path2"/n) = 1 ]] diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 922162d4b..494b24ddb 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -166,7 +166,7 @@ printf 4.0 > "$flake1Dir"/version printf Utrecht > "$flake1Dir"/who nix profile add "$flake1Dir" [[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[[ $(nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -r .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' # Override the outputs. nix profile remove simple flake1 diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 2893efec7..1bcaf2f53 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -58,7 +58,7 @@ nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' +nix path-info --json "$outPathCA" | jq -e '.[].ca | .method == "flat" and .hash.algorithm == "md5"' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed.