nlohmann::json instance and JSON Schema for Hash

Improving and codifying our experimental JSON interfacing.

Co-Authored-By: Robert Hensing <robert@roberthensing.nl>

doc/manual/meson.build

@@ -88,7 +88,7 @@ manual = custom_target(
         @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
         @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
         sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
-        @4@ -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
+        @4@ -r -L --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
         (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
         rm -rf @2@/manual
         mv @2@/html @2@/manual

doc/manual/package.nix

@@ -33,6 +33,8 @@ mkMesonDerivation (finalAttrs: {
     fileset.difference
       (fileset.unions [
         ../../.version
+        # For example JSON
+        ../../src/libutil-tests/data/hash
         # Too many different types of files to filter for now
         ../../doc/manual
         ./.

doc/manual/source/protocols/json/hash.md

@@ -1,5 +1,31 @@
 {{#include hash-v1-fixed.md}}
 
+## Examples
+
+### SHA-256 with Base64 encoding
+
+```json
+{{#include schema/hash-v1/sha256-base64.json}}
+```
+
+### SHA-256 with Base16 (hexadecimal) encoding
+
+```json
+{{#include schema/hash-v1/sha256-base16.json}}
+```
+
+### SHA-256 with Nix32 encoding
+
+```json
+{{#include schema/hash-v1/sha256-nix32.json}}
+```
+
+### BLAKE3 with Base64 encoding
+
+```json
+{{#include schema/hash-v1/blake3-base64.json}}
+```
+
 <!--
 ## Raw Schema
 

doc/manual/source/protocols/json/schema/hash-v1

@@ -0,0 +1 @@
+../../../../../../src/libutil-tests/data/hash/

doc/manual/source/protocols/json/schema/hash-v1.yaml

@@ -5,15 +5,38 @@ description: |
   A cryptographic hash value used throughout Nix for content addressing and integrity verification.
 
   This schema describes the JSON representation of Nix's `Hash` type.
-
-  TODO Work in progress
 type: object
 properties:
   algorithm:
-    title: Hash algorithm
     "$ref": "#/$defs/algorithm"
+  format:
+    type: string
+    enum:
+    - base64
+    - nix32
+    - base16
+    - sri
+    title: Hash format
+    description: |
+      The encoding format of the hash value.
+
+      - `base64` uses standard Base64 encoding [RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4)
+      - `nix32` is Nix-specific base-32 encoding
+      - `base16` is lowercase hexadecimal
+      - `sri` is the [Subresource Integrity format](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity).
+  hash:
+    type: string
+    title: Hash
+    description: |
+      The encoded hash value, itself.
+
+      It is specified in the format specified by the `format` field.
+      It must be the right length for the hash algorithm specified in the `algorithm` field, also.
+      The hash value does not include any algorithm prefix.
 required:
 - algorithm
+- format
+- hash
 additionalProperties: false
 "$defs":
   algorithm:
@@ -24,6 +47,7 @@ additionalProperties: false
     - sha1
     - sha256
     - sha512
+    title: Hash algorithm
     description: |
       The hash algorithm used to compute the hash value.
 

src/json-schema-checks/hash

@@ -0,0 +1 @@
+../../src/libutil-tests/data/hash

src/json-schema-checks/meson.build

@@ -20,6 +20,16 @@ schema_dir = meson.current_source_dir() / 'schema'
 
 # Get all example files
 schemas = [
+  {
+    'stem' : 'hash',
+    'schema' : schema_dir / 'hash-v1.yaml',
+    'files' : [
+      'sha256-base64.json',
+      'sha256-base16.json',
+      'sha256-nix32.json',
+      'blake3-base64.json',
+    ],
+  },
   {
     'stem' : 'derivation',
     'schema' : schema_dir / 'derivation-v3.yaml',

src/json-schema-checks/package.nix

@@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: {
   fileset = lib.fileset.unions [
     ../../.version
     ../../doc/manual/source/protocols/json/schema
+    ../../src/libutil-tests/data/hash
     ../../src/libstore-tests/data/derivation
     ./.
   ];

src/libutil-tests/data/hash/blake3-base64.json

@@ -0,0 +1,5 @@
+{
+    "algorithm": "blake3",
+    "format": "base64",
+    "hash": "nnDuFEmWX7YtBJBAoe0G7Dd0MNpuwTFz58T//NKL6YA="
+}

src/libutil-tests/data/hash/sha256-base16.json

@@ -0,0 +1,5 @@
+{
+    "algorithm": "sha256",
+    "format": "base16",
+    "hash": "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"
+}

src/libutil-tests/data/hash/sha256-base64.json

@@ -0,0 +1,5 @@
+{
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
+}

src/libutil-tests/data/hash/sha256-nix32.json

@@ -0,0 +1,5 @@
+{
+    "algorithm": "sha256",
+    "format": "nix32",
+    "hash": "0fz12qc1nillhvhw6bvs4ka18789x8dqaipjb316x4aqdkvw5r7h"
+}

src/libutil-tests/data/hash/simple.json

@@ -0,0 +1,5 @@
+{
+  "algorithm": "sha256",
+  "format": "base64",
+  "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
+}

src/libutil-tests/hash.cc

@@ -4,30 +4,30 @@
 #include <nlohmann/json.hpp>
 
 #include "nix/util/hash.hh"
-#include "nix/util/tests/characterization.hh"
+#include "nix/util/tests/json-characterization.hh"
 
 namespace nix {
 
-class HashTest : public CharacterizationTest
+class HashTest : public virtual CharacterizationTest
 {
     std::filesystem::path unitTestData = getUnitTestData() / "hash";
 
 public:
 
-    /**
-     * We set these in tests rather than the regular globals so we don't have
-     * to worry about race conditions if the tests run concurrently.
-     */
-    ExperimentalFeatureSettings mockXpSettings;
-
     std::filesystem::path goldenMaster(std::string_view testStem) const override
     {
         return unitTestData / testStem;
     }
 };
 
-class BLAKE3HashTest : public HashTest
+struct BLAKE3HashTest : virtual HashTest
 {
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+
     void SetUp() override
     {
         mockXpSettings.set("experimental-features", "blake3-hashes");
@@ -203,4 +203,97 @@ TEST(hashFormat, testParseHashFormatOptException)
 {
     ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt);
 }
+
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+using nlohmann::json;
+
+struct HashJsonTest : virtual HashTest,
+                      JsonCharacterizationTest<Hash>,
+                      ::testing::WithParamInterface<std::pair<std::string_view, Hash>>
+{};
+
+struct HashJsonParseOnlyTest : virtual HashTest,
+                               JsonCharacterizationTest<Hash>,
+                               ::testing::WithParamInterface<std::pair<std::string_view, Hash>>
+{};
+
+struct BLAKE3HashJsonTest : virtual HashTest,
+                            BLAKE3HashTest,
+                            JsonCharacterizationTest<Hash>,
+                            ::testing::WithParamInterface<std::pair<std::string_view, Hash>>
+{};
+
+TEST_P(HashJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    readJsonTest(name, expected);
+}
+
+TEST_P(HashJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+TEST_P(HashJsonParseOnlyTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    readJsonTest(name, expected);
+}
+
+TEST_P(BLAKE3HashJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    readJsonTest(name, expected, mockXpSettings);
+}
+
+TEST_P(BLAKE3HashJsonTest, to_json)
+{
+    auto & [name, expected] = GetParam();
+    writeJsonTest(name, expected);
+}
+
+// Round-trip tests (from_json + to_json) for base64 format only
+// (to_json always outputs base64)
+INSTANTIATE_TEST_SUITE_P(
+    HashJSON,
+    HashJsonTest,
+    ::testing::Values(
+        std::pair{
+            "simple",
+            hashString(HashAlgorithm::SHA256, "asdf"),
+        },
+        std::pair{
+            "sha256-base64",
+            hashString(HashAlgorithm::SHA256, "asdf"),
+        }));
+
+// Parse-only tests for non-base64 formats
+// These verify C++ can deserialize other formats correctly
+INSTANTIATE_TEST_SUITE_P(
+    HashJSONParseOnly,
+    HashJsonParseOnlyTest,
+    ::testing::Values(
+        std::pair{
+            "sha256-base16",
+            hashString(HashAlgorithm::SHA256, "asdf"),
+        },
+        std::pair{
+            "sha256-nix32",
+            hashString(HashAlgorithm::SHA256, "asdf"),
+        }));
+
+INSTANTIATE_TEST_SUITE_P(BLAKE3HashJSONParseOnly, BLAKE3HashJsonTest, ([] {
+                             ExperimentalFeatureSettings mockXpSettings;
+                             mockXpSettings.set("experimental-features", "blake3-hashes");
+                             return ::testing::Values(
+                                 std::pair{
+                                     "blake3-base64",
+                                     hashString(HashAlgorithm::BLAKE3, "asdf", mockXpSettings),
+                                 });
+                         }()));
+
 } // namespace nix

src/libutil/hash.cc

@@ -13,6 +13,7 @@
 #include "nix/util/split.hh"
 #include "nix/util/base-n.hh"
 #include "nix/util/base-nix-32.hh"
+#include "nix/util/json-utils.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -141,9 +142,13 @@ static HashFormat baseFromSize(std::string_view rest, HashAlgorithm algo)
  *
  * @param rest the string view to parse. Must not include any `<algo>(:|-)` prefix.
  */
-static Hash parseLowLevel(std::string_view rest, HashAlgorithm algo, DecodeNamePair pair)
+static Hash parseLowLevel(
+    std::string_view rest,
+    HashAlgorithm algo,
+    DecodeNamePair pair,
+    const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
-    Hash res{algo};
+    Hash res{algo, xpSettings};
     std::string d;
     try {
         d = pair.decode(rest);
@@ -244,9 +249,10 @@ Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
     return parseExplicitFormatUnprefixed(s, algo, baseFromSize(s, algo));
 }
 
-Hash Hash::parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat format)
+Hash Hash::parseExplicitFormatUnprefixed(
+    std::string_view s, HashAlgorithm algo, HashFormat format, const ExperimentalFeatureSettings & xpSettings)
 {
-    return parseLowLevel(s, algo, baseExplicit(format));
+    return parseLowLevel(s, algo, baseExplicit(format), xpSettings);
 }
 
 Hash Hash::random(HashAlgorithm algo)
@@ -446,10 +452,12 @@ std::string_view printHashFormat(HashFormat HashFormat)
     }
 }
 
-std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
+std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s, const ExperimentalFeatureSettings & xpSettings)
 {
-    if (s == "blake3")
+    if (s == "blake3") {
+        xpSettings.require(Xp::BLAKE3Hashes);
         return HashAlgorithm::BLAKE3;
+    }
     if (s == "md5")
         return HashAlgorithm::MD5;
     if (s == "sha1")
@@ -461,9 +469,9 @@ std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
     return std::nullopt;
 }
 
-HashAlgorithm parseHashAlgo(std::string_view s)
+HashAlgorithm parseHashAlgo(std::string_view s, const ExperimentalFeatureSettings & xpSettings)
 {
-    auto opt_h = parseHashAlgoOpt(s);
+    auto opt_h = parseHashAlgoOpt(s, xpSettings);
     if (opt_h)
         return *opt_h;
     else
@@ -491,3 +499,27 @@ std::string_view printHashAlgo(HashAlgorithm ha)
 }
 
 } // namespace nix
+
+namespace nlohmann {
+
+using namespace nix;
+
+Hash adl_serializer<Hash>::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings)
+{
+    auto & obj = getObject(json);
+    auto algo = parseHashAlgo(getString(valueAt(obj, "algorithm")), xpSettings);
+    auto format = parseHashFormat(getString(valueAt(obj, "format")));
+    auto & hashS = getString(valueAt(obj, "hash"));
+    return Hash::parseExplicitFormatUnprefixed(hashS, algo, format, xpSettings);
+}
+
+void adl_serializer<Hash>::to_json(json & json, const Hash & hash)
+{
+    json = {
+        {"format", printHashFormat(HashFormat::Base64)},
+        {"algorithm", printHashAlgo(hash.algo)},
+        {"hash", hash.to_string(HashFormat::Base64, false)},
+    };
+}
+
+} // namespace nlohmann

src/libutil/include/nix/util/hash.hh

@@ -5,6 +5,7 @@
 #include "nix/util/types.hh"
 #include "nix/util/serialise.hh"
 #include "nix/util/file-system.hh"
+#include "nix/util/json-impls.hh"
 
 namespace nix {
 
@@ -97,7 +98,11 @@ struct Hash
      * @param explicitFormat cannot be SRI, but must be one of the
      * "bases".
      */
-    static Hash parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat explicitFormat);
+    static Hash parseExplicitFormatUnprefixed(
+        std::string_view s,
+        HashAlgorithm algo,
+        HashFormat explicitFormat,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     static Hash parseSRI(std::string_view original);
 
@@ -188,12 +193,14 @@ std::string_view printHashFormat(HashFormat hashFormat);
 /**
  * Parse a string representing a hash algorithm.
  */
-HashAlgorithm parseHashAlgo(std::string_view s);
+HashAlgorithm
+parseHashAlgo(std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
 /**
  * Will return nothing on parse error
  */
-std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s);
+std::optional<HashAlgorithm>
+parseHashAlgoOpt(std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
 /**
  * And the reverse.
@@ -221,6 +228,10 @@ public:
     HashResult currentHash();
 };
 
+template<>
+struct json_avoids_null<Hash> : std::true_type
+{};
+
 } // namespace nix
 
 template<>
@@ -241,3 +252,5 @@ inline std::size_t hash_value(const Hash & hash)
 }
 
 } // namespace nix
+
+JSON_IMPL_WITH_XP_FEATURES(Hash)