Make the store path info ca field structured in JSON

The old string format is a holdover from the pre JSON days. It is not
friendly to users who need to get the information out of it.

Also introduce the sort of versioning we have for derivation for this
format too.

doc/manual/source/protocols/json/meson.build

@@ -12,7 +12,7 @@ schemas = [
   'hash-v1',
   'content-address-v1',
   'store-path-v1',
-  'store-object-info-v1',
+  'store-object-info-v2',
   'derivation-v4',
   'deriving-path-v1',
   'build-trace-entry-v1',

doc/manual/source/protocols/json/schema/store-object-info-v2.yaml

@@ -1,6 +1,6 @@
-"$schema": "http://json-schema.org/draft-07/schema"
-"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v1.json"
-title: Store Object Info
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json"
+title: Store Object Info v2
 description: |
   Information about a [store object](@docroot@/store/store-object.md).
 
@@ -41,11 +41,27 @@ $defs:
       This is the minimal set of fields that describe what a store object contains.
     type: object
     required:
+      - version
       - narHash
       - narSize
       - references
       - ca
     properties:
+      version:
+        type: integer
+        const: 2
+        title: Format version (must be 2)
+        description: |
+          Must be `2`.
+          This is a guard that allows us to continue evolving this format.
+          Here is the rough version history:
+
+          - Version 0: `.narinfo` line-oriented format
+
+          - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from `.narinfo` format.
+
+          - Version 2: Use structured JSON type for `ca`
+
       path:
         type: string
         title: Store Path
@@ -76,7 +92,10 @@ $defs:
           type: string
 
       ca:
-        type: ["string", "null"]
+        oneOf:
+          - type: "null"
+            const: null
+          - "$ref": "./content-address-v1.yaml"
         title: Content Address
         description: |
           If the store object is [content-addressed](@docroot@/store/store-object/content-address.md),
@@ -91,6 +110,7 @@ $defs:
       In other words, the same store object in different stores could have different values for these impure fields.
     type: object
     required:
+      - version
       - narHash
       - narSize
       - references
@@ -101,6 +121,7 @@ $defs:
       - ultimate
       - signatures
     properties:
+      version: { $ref: "#/$defs/base/properties/version" }
       path: { $ref: "#/$defs/base/properties/path" }
       narHash: { $ref: "#/$defs/base/properties/narHash" }
       narSize: { $ref: "#/$defs/base/properties/narSize" }
@@ -164,6 +185,7 @@ $defs:
       This download information, being specific to how the store object happens to be stored and transferred, is also considered to be non-intrinsic / impure.
     type: object
     required:
+      - version
       - narHash
       - narSize
       - references
@@ -179,6 +201,7 @@ $defs:
       - downloadHash
       - downloadSize
     properties:
+      version: { $ref: "#/$defs/base/properties/version" }
       path: { $ref: "#/$defs/base/properties/path" }
       narHash: { $ref: "#/$defs/base/properties/narHash" }
       narSize: { $ref: "#/$defs/base/properties/narSize" }

doc/manual/source/protocols/json/store-object-info.md

@@ -1,29 +1,29 @@
-{{#include store-object-info-v1-fixed.md}}
+{{#include store-object-info-v2-fixed.md}}
 
 ## Examples
 
 ### Minimal store object (content-addressed)
 
 ```json
-{{#include schema/store-object-info-v1/pure.json}}
+{{#include schema/store-object-info-v2/pure.json}}
 ```
 
 ### Store object with impure fields
 
 ```json
-{{#include schema/store-object-info-v1/impure.json}}
+{{#include schema/store-object-info-v2/impure.json}}
 ```
 
 ### Minimal store object (empty)
 
 ```json
-{{#include schema/store-object-info-v1/empty_pure.json}}
+{{#include schema/store-object-info-v2/empty_pure.json}}
 ```
 
 ### Store object with all impure fields
 
 ```json
-{{#include schema/store-object-info-v1/empty_impure.json}}
+{{#include schema/store-object-info-v2/empty_impure.json}}
 ```
 
 ### NAR info (minimal)
@@ -41,5 +41,5 @@
 <!-- need to convert YAML to JSON first
 ## Raw Schema
 
-[JSON Schema for Store Object Info v1](schema/store-object-info-v1.json)
+[JSON Schema for Store Object Info v1](schema/store-object-info-v2.json)
 -->

src/json-schema-checks/meson.build

@@ -134,7 +134,7 @@ schemas += [
   # Match overall
   {
     'stem' : 'store-object-info',
-    'schema' : schema_dir / 'store-object-info-v1.yaml',
+    'schema' : schema_dir / 'store-object-info-v2.yaml',
     'files' : [
       'pure.json',
       'impure.json',
@@ -144,7 +144,7 @@ schemas += [
   },
   {
     'stem' : 'nar-info',
-    'schema' : schema_dir / 'store-object-info-v1.yaml',
+    'schema' : schema_dir / 'store-object-info-v2.yaml',
     'files' : [
       'pure.json',
       'impure.json',
@@ -162,7 +162,7 @@ schemas += [
   # Match exact variant
   {
     'stem' : 'store-object-info',
-    'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base',
+    'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base',
     'files' : [
       'pure.json',
       'empty_pure.json',
@@ -170,7 +170,7 @@ schemas += [
   },
   {
     'stem' : 'store-object-info',
-    'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/impure',
+    'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure',
     'files' : [
       'impure.json',
       'empty_impure.json',
@@ -178,14 +178,14 @@ schemas += [
   },
   {
     'stem' : 'nar-info',
-    'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base',
+    'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base',
     'files' : [
       'pure.json',
     ],
   },
   {
     'stem' : 'nar-info',
-    'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/narInfo',
+    'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo',
     'files' : [
       'impure.json',
     ],

src/libstore-tests/data/nar-info/impure.json

@@ -1,5 +1,12 @@
 {
-  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "ca": {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+    },
+    "method": "nar"
+  },
   "compression": "xz",
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
   "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
@@ -16,5 +23,6 @@
     "qwer"
   ],
   "ultimate": true,
-  "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"
+  "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz",
+  "version": 2
 }

src/libstore-tests/data/nar-info/pure.json

@@ -1,9 +1,17 @@
 {
-  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "ca": {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+    },
+    "method": "nar"
+  },
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
-  ]
+  ],
+  "version": 2
 }

src/libstore-tests/data/path-info/empty_impure.json

@@ -6,5 +6,6 @@
   "references": [],
   "registrationTime": null,
   "signatures": [],
-  "ultimate": false
+  "ultimate": false,
+  "version": 2
 }

src/libstore-tests/data/path-info/empty_pure.json

@@ -2,5 +2,6 @@
   "ca": null,
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 0,
-  "references": []
+  "references": [],
+  "version": 2
 }

src/libstore-tests/data/path-info/impure.json

@@ -1,5 +1,12 @@
 {
-  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "ca": {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+    },
+    "method": "nar"
+  },
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
@@ -12,5 +19,6 @@
     "asdf",
     "qwer"
   ],
-  "ultimate": true
+  "ultimate": true,
+  "version": 2
 }

src/libstore-tests/data/path-info/pure.json

@@ -1,9 +1,17 @@
 {
-  "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh",
+  "ca": {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+    },
+    "method": "nar"
+  },
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
-  ]
+  ],
+  "version": 2
 }

src/libstore/path-info.cc

@@ -156,6 +156,8 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf
 
     auto jsonObject = json::object();
 
+    jsonObject["version"] = 2;
+
     jsonObject["narHash"] = narHash.to_string(hashFormat, true);
     jsonObject["narSize"] = narSize;
 
@@ -165,7 +167,7 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf
             jsonRefs.emplace_back(store.printStorePath(ref));
     }
 
-    jsonObject["ca"] = ca ? (std::optional{renderContentAddress(*ca)}) : std::nullopt;
+    jsonObject["ca"] = ca;
 
     if (includeImpureInfo) {
         jsonObject["deriver"] = deriver ? (std::optional{store.printStorePath(*deriver)}) : std::nullopt;
@@ -189,6 +191,16 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store
     };
 
     auto & json = getObject(_json);
+
+    // Check version (optional for backward compatibility)
+    nlohmann::json::number_unsigned_t version = 1;
+    if (json.contains("version")) {
+        version = getUnsigned(valueAt(json, "version"));
+        if (version != 1 && version != 2) {
+            throw Error("Unsupported path info JSON format version %d, expected 1 through 2", version);
+        }
+    }
+
     res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt);
     res.narSize = getUnsigned(valueAt(json, "narSize"));
 
@@ -205,7 +217,15 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store
     // missing is for back-compat.
     if (auto * rawCa0 = optionalValueAt(json, "ca"))
         if (auto * rawCa = getNullable(*rawCa0))
-            res.ca = ContentAddress::parse(getString(*rawCa));
+            switch (version) {
+            case 1:
+                // old string format also used in SQLite DB and .narinfo
+                res.ca = ContentAddress::parse(getString(*rawCa));
+                break;
+            case 2 ... std::numeric_limits<decltype(version)>::max():
+                res.ca = *rawCa;
+                break;
+            }
 
     if (auto * rawDeriver0 = optionalValueAt(json, "deriver"))
         if (auto * rawDeriver = getNullable(*rawDeriver0))

tests/functional/fixed.sh

@@ -14,7 +14,16 @@ nix-build fixed.nix -A bad --no-out-link && fail "should fail"
 # Building with the bad hash should produce the "good" output path as
 # a side-effect.
 [[ -e $path ]]
-nix path-info --json "$path" | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd
+nix path-info --json "$path" | jq -e \
+    --arg hash "$(nix hash convert --to base64 "md5:8ddd8be4b179a529afa5f2ffae4b9858")" \
+    '.[].ca == {
+        method: "flat",
+        hash: {
+            algorithm: "md5",
+            format: "base64",
+            hash: $hash
+        },
+    }'
 
 echo 'testing good...'
 nix-build fixed.nix -A good --no-out-link

tests/functional/git-hashing/simple-common.sh

@@ -47,9 +47,17 @@ try2 () {
     hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath")
     [[ "$hashFromGit" == "$expected" ]]
 
-    local caFromNix
-    caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca")
-    [[ "fixed:git:$hashAlgo:$(nix hash convert --to nix32 "$hashAlgo:$hashFromGit")" = "$caFromNix" ]]
+    nix path-info --json "$path" | jq -e \
+        --arg algo "$hashAlgo" \
+        --arg hash "$(nix hash convert --to base64 "$hashAlgo:$hashFromGit")" \
+        '.[].ca == {
+            method: "git",
+            hash: {
+                algorithm: $algo,
+                format: "base64",
+                hash: $hash
+            },
+        }'
 }
 
 test0 () {

tests/functional/impure-derivations.sh

@@ -30,7 +30,7 @@ path1_stuff=$(echo "$json" | jq -r .[].outputs.stuff)
 [[ $(< "$path1"/n) = 0 ]]
 [[ $(< "$path1_stuff"/bla) = 0 ]]
 
-[[ $(nix path-info --json "$path1" | jq .[].ca) =~ fixed:r:sha256: ]]
+nix path-info --json "$path1" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"'
 
 path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
 [[ $(< "$path2"/n) = 1 ]]

tests/functional/nix-profile.sh

@@ -166,7 +166,7 @@ printf 4.0 > "$flake1Dir"/version
 printf Utrecht > "$flake1Dir"/who
 nix profile add "$flake1Dir"
 [[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Utrecht" ]]
-[[ $(nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -r .[].ca) =~ fixed:r:sha256: ]]
+nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"'
 
 # Override the outputs.
 nix profile remove simple flake1

tests/functional/signing.sh

@@ -58,7 +58,7 @@ nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1"
 # Build something content-addressed.
 outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link)
 
-nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")'
+nix path-info --json "$outPathCA" | jq -e '.[].ca | .method == "flat" and .hash.algorithm == "md5"'
 
 # Content-addressed paths don't need signatures, so they verify
 # regardless of --sigs-needed.