nlohmann::json instance and JSON Schema for ContentAddress

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>

doc/manual/package.nix

@@ -35,6 +35,7 @@ mkMesonDerivation (finalAttrs: {
         ../../.version
         # For example JSON
         ../../src/libutil-tests/data/hash
+        ../../src/libstore-tests/data/content-address
         # Too many different types of files to filter for now
         ../../doc/manual
         ./.

doc/manual/source/SUMMARY.md.in

@@ -118,6 +118,7 @@
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
     - [Hash](protocols/json/hash.md)
+    - [Pseudo Content Address](protocols/json/content-address.md)
     - [Store Object Info](protocols/json/store-object-info.md)
     - [Derivation](protocols/json/derivation.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)

doc/manual/source/protocols/json/content-address.md

@@ -0,0 +1,21 @@
+{{#include content-address-v1-fixed.md}}
+
+## Examples
+
+### [Text](file:///home/jcericson/src/nix/4/build-linux-clang/src/nix-manual/manual/store/store-object/content-address.html#method-text) method
+
+```json
+{{#include schema/content-address-v1/text.json}}
+```
+
+### [Nix Archive](file:///home/jcericson/src/nix/4/build-linux-clang/src/nix-manual/manual/store/store-object/content-address.html#method-nix-archive) method
+
+```json
+{{#include schema/content-address-v1/nar.json}}
+```
+
+<!-- need to convert YAML to JSON first
+## Raw Schema
+
+[JSON Schema for Hash v1](schema/content-address-v1.json)
+-->

doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed

@@ -12,3 +12,6 @@ s/\\`/`/g
 # As we have more such relative links, more replacements of this nature
 # should appear below.
 s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
+s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
+s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
+s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g

doc/manual/source/protocols/json/meson.build

@@ -10,6 +10,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml')
 
 schemas = [
   'hash-v1',
+  'content-address-v1',
   'derivation-v3',
 ]
 

doc/manual/source/protocols/json/schema/content-address-v1

@@ -0,0 +1 @@
+../../../../../../src/libstore-tests/data/content-address

doc/manual/source/protocols/json/schema/content-address-v1.yaml

@@ -0,0 +1,51 @@
+"$schema": http://json-schema.org/draft-04/schema#
+"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/content-address-v1.json
+title: Pesudo Content Address
+description: |
+  This schema describes the JSON representation of Nix's `ContentAddress` type.
+
+  This data type is not a simple as a looks, and therefore called a *pseudo* content address.
+  See the description of the `hash` field for why this is.
+
+  When creating the store path of a content-addressed store object, the `hash` from this will be combined with the references of the store object in order to create a content-address that is properly sensitive to the entirety of the store object — file system objects and references alike.
+  So it is that store path that is arguably the *true* content-address for content-addressed store paths, not this data type.
+
+  The only problem with that is that store paths are truncated in various ways from the underlying hashes, so their cryptographic strength can be reasonably doubted.
+  Given that uncertain cryptographic strength, we do need something else.
+  Currently this in conjunction with the reference list captures all the content securely, more concisely.
+  (Though the reference list is still arbitrarily-sized, it is presumably in practice still way smaller than the file system object data).
+  
+  > **Note**
+  >
+  > A hypothetical hash of both of those which is *not* so truncated would be even better, as it would be secure and fixed-size.
+  > A hypothetical new method content-addressing store objects where we compute the store paths just from such a hash, so we don't need to "dereference" the hash to get the underlying reference list and rehash it, would be better still.
+type: object
+properties:
+  method:
+    "$ref": "#/$defs/method"
+  hash:
+    title: Content Address
+    description: |
+      This would be the content-address itself.
+
+      For all current methods, this is just a content address of the file system object of the store object, [as described in the store chapter](@docroot@/store/store-object/content-address.md), and not of the store object as a whole.
+      In particular, the references of the store object are *not* taken into account with this hash (and currently-supported methods).
+    "$ref": "./hash-v1.yaml"
+required:
+- method
+- hash
+additionalProperties: false
+"$defs":
+  method:
+    type: string
+    enum: [flat, nar, text, git]
+    title: Content-Addressing Method
+    description: |
+      A string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
+
+      Valid method strings are:
+
+      - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) (provided the contents are a single file)
+      - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
+      - [`text`](@docroot@/store/store-object/content-address.md#method-text)
+      - [`git`](@docroot@/store/store-object/content-address.md#method-git)

doc/manual/source/protocols/json/schema/derivation-v3.yaml

@@ -154,19 +154,10 @@ properties:
           The output path, if known in advance.
 
       method:
-        type: string
-        title: Content addressing method
-        enum: [flat, nar, text, git]
+        "$ref": "./content-address-v1.yaml#/$defs/method"
         description: |
           For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
-
-          Valid method strings are:
-
-          - [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
-          - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
-          - [`text`](@docroot@/store/store-object/content-address.md#method-text)
-          - [`git`](@docroot@/store/store-object/content-address.md#method-git)
-
+          See the linked original definition for further details.
       hashAlgo:
         title: Hash algorithm
         "$ref": "./hash-v1.yaml#/$defs/algorithm"

src/json-schema-checks/content-address

@@ -0,0 +1 @@
+../../src/libstore-tests/data/content-address

src/json-schema-checks/meson.build

@@ -30,6 +30,14 @@ schemas = [
       'blake3-base64.json',
     ],
   },
+  {
+    'stem' : 'content-address',
+    'schema' : schema_dir / 'content-address-v1.yaml',
+    'files' : [
+      'text.json',
+      'nar.json',
+    ],
+  },
   {
     'stem' : 'derivation',
     'schema' : schema_dir / 'derivation-v3.yaml',
@@ -64,8 +72,6 @@ foreach schema : schemas
       stem + '-schema-valid',
       jv,
       args : [
-        '--map',
-        './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml',
         'http://json-schema.org/draft-04/schema',
         schema_file,
       ],

src/json-schema-checks/package.nix

@@ -21,6 +21,7 @@ mkMesonDerivation (finalAttrs: {
     ../../.version
     ../../doc/manual/source/protocols/json/schema
     ../../src/libutil-tests/data/hash
+    ../../src/libstore-tests/data/content-address
     ../../src/libstore-tests/data/derivation
     ./.
   ];

src/libstore-tests/content-address.cc

@@ -1,6 +1,7 @@
 #include <gtest/gtest.h>
 
 #include "nix/store/content-address.hh"
+#include "nix/util/tests/json-characterization.hh"
 
 namespace nix {
 
@@ -8,33 +9,93 @@ namespace nix {
  * ContentAddressMethod::parse, ContentAddressMethod::render
  * --------------------------------------------------------------------------*/
 
-TEST(ContentAddressMethod, testRoundTripPrintParse_1)
+static auto methods = ::testing::Values(
+    std::pair{ContentAddressMethod::Raw::Text, "text"},
+    std::pair{ContentAddressMethod::Raw::Flat, "flat"},
+    std::pair{ContentAddressMethod::Raw::NixArchive, "nar"},
+    std::pair{ContentAddressMethod::Raw::Git, "git"});
+
+struct ContentAddressMethodTest : ::testing::Test,
+                                  ::testing::WithParamInterface<std::pair<ContentAddressMethod, std::string_view>>
+{};
+
+TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1)
 {
-    for (ContentAddressMethod cam : {
-             ContentAddressMethod::Raw::Text,
-             ContentAddressMethod::Raw::Flat,
-             ContentAddressMethod::Raw::NixArchive,
-             ContentAddressMethod::Raw::Git,
-         }) {
-        EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
-    }
+    auto & [cam, _] = GetParam();
+    EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
 }
 
-TEST(ContentAddressMethod, testRoundTripPrintParse_2)
+TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2)
 {
-    for (const std::string_view camS : {
-             "text",
-             "flat",
-             "nar",
-             "git",
-         }) {
-        EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
-    }
+    auto & [cam, camS] = GetParam();
+    EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
 }
 
+INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods);
+
 TEST(ContentAddressMethod, testParseContentAddressMethodOptException)
 {
     EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError);
 }
 
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+class ContentAddressTest : public virtual CharacterizationTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "content-address";
+
+public:
+
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
+
+using nlohmann::json;
+
+struct ContentAddressJsonTest : ContentAddressTest,
+                                JsonCharacterizationTest<ContentAddress>,
+                                ::testing::WithParamInterface<std::pair<std::string_view, ContentAddress>>
+{};
+
+TEST_P(ContentAddressJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    readJsonTest(name, expected);
+}
+
+TEST_P(ContentAddressJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    ContentAddressJSON,
+    ContentAddressJsonTest,
+    ::testing::Values(
+        std::pair{
+            "text",
+            ContentAddress{
+                .method = ContentAddressMethod::Raw::Text,
+                .hash = hashString(HashAlgorithm::SHA256, "asdf"),
+            },
+        },
+        std::pair{
+            "nar",
+            ContentAddress{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hash = hashString(HashAlgorithm::SHA256, "qwer"),
+            },
+        }));
+
 } // namespace nix

src/libstore-tests/data/content-address/nar.json

@@ -0,0 +1,8 @@
+{
+  "hash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "9vLqj0XYoFfJVmoz+ZR02i5camYE1zYSFlDicwxvsKM="
+  },
+  "method": "nar"
+}

src/libstore-tests/data/content-address/text.json

@@ -0,0 +1,8 @@
+{
+  "hash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
+  },
+  "method": "text"
+}

src/libstore/content-address.cc

@@ -1,6 +1,7 @@
 #include "nix/util/args.hh"
 #include "nix/store/content-address.hh"
 #include "nix/util/split.hh"
+#include "nix/util/json-utils.hh"
 
 namespace nix {
 
@@ -300,3 +301,36 @@ Hash ContentAddressWithReferences::getHash() const
 }
 
 } // namespace nix
+
+namespace nlohmann {
+
+using namespace nix;
+
+ContentAddressMethod adl_serializer<ContentAddressMethod>::from_json(const json & json)
+{
+    return ContentAddressMethod::parse(getString(json));
+}
+
+void adl_serializer<ContentAddressMethod>::to_json(json & json, const ContentAddressMethod & m)
+{
+    json = m.render();
+}
+
+ContentAddress adl_serializer<ContentAddress>::from_json(const json & json)
+{
+    auto obj = getObject(json);
+    return {
+        .method = adl_serializer<ContentAddressMethod>::from_json(valueAt(obj, "method")),
+        .hash = valueAt(obj, "hash"),
+    };
+}
+
+void adl_serializer<ContentAddress>::to_json(json & json, const ContentAddress & ca)
+{
+    json = {
+        {"method", ca.method},
+        {"hash", ca.hash},
+    };
+}
+
+} // namespace nlohmann

src/libstore/include/nix/store/content-address.hh

@@ -6,6 +6,7 @@
 #include "nix/store/path.hh"
 #include "nix/util/file-content-address.hh"
 #include "nix/util/variant-wrapper.hh"
+#include "nix/util/json-impls.hh"
 
 namespace nix {
 
@@ -308,4 +309,15 @@ struct ContentAddressWithReferences
     Hash getHash() const;
 };
 
+template<>
+struct json_avoids_null<ContentAddressMethod> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<ContentAddress> : std::true_type
+{};
+
 } // namespace nix
+
+JSON_IMPL(nix::ContentAddressMethod)
+JSON_IMPL(nix::ContentAddress)