JSON impl and Schema for DummyStore

This is the "keystone" that puts most of the other store-layer JSON
formats together.

doc/manual/package.nix

@@ -46,6 +46,7 @@ mkMesonDerivation (finalAttrs: {
         ../../src/libstore-tests/data/path-info
         ../../src/libstore-tests/data/nar-info
         ../../src/libstore-tests/data/build-result
+        ../../src/libstore-tests/data/dummy-store
         # Too many different types of files to filter for now
         ../../doc/manual
         ./.

doc/manual/source/SUMMARY.md.in

@@ -130,6 +130,7 @@
     - [Deriving Path](protocols/json/deriving-path.md)
     - [Build Trace Entry](protocols/json/build-trace-entry.md)
     - [Build Result](protocols/json/build-result.md)
+    - [Store](protocols/json/store.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Store Path Specification](protocols/store-path.md)
   - [Nix Archive (NAR) Format](protocols/nix-archive/index.md)

doc/manual/source/protocols/json/meson.build

@@ -18,6 +18,7 @@ schemas = [
   'deriving-path-v1',
   'build-trace-entry-v1',
   'build-result-v1',
+  'store-v1',
 ]
 
 schema_files = files()

doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml

@@ -4,20 +4,37 @@ title: Build Trace Entry
 description: |
   A record of a successful build outcome for a specific derivation output.
 
-  This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md) entry.
+  This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md).
 
   > **Warning**
   >
   > This JSON format is currently
   > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
   > and subject to change.
-
-type: object
 required:
   - id
   - outPath
   - dependentRealisations
   - signatures
+allOf:
+  - "$ref": "#/$defs/key"
+  - "$ref": "#/$defs/value"
+properties:
+  id: {}
+  outPath: {}
+  dependentRealisations: {}
+  signatures: {}
+additionalProperties: false
+
+"$defs":
+  key:
+    title: Build Trace Key
+    description: |
+      A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair.
+      This is the "key" part, refering to a derivation and output.
+    type: object
+    required:
+      - id
     properties:
       id:
         type: string
@@ -35,6 +52,17 @@ properties:
 
           Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"`
 
+  value:
+    title: Build Trace Value
+    description: |
+      A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair.
+      This is the "value" part, describing an output.
+    type: object
+    required:
+      - outPath
+      - dependentRealisations
+      - signatures
+    properties:
       outPath:
         "$ref": "store-path-v1.yaml"
         title: Output Store Path
@@ -56,7 +84,7 @@ properties:
 
         patternProperties:
           "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
-        $ref: "store-path-v1.yaml"
+            "$ref": "store-path-v1.yaml"
             title: Dependent Store Path
             description: Store path that this dependency resolved to during the build
         additionalProperties: false
@@ -70,5 +98,3 @@ properties:
           type: string
           title: Signature
           description: A single cryptographic signature
-
-additionalProperties: false

doc/manual/source/protocols/json/schema/store-v1

@@ -0,0 +1 @@
+../../../../../../src/libstore-tests/data/dummy-store

doc/manual/source/protocols/json/schema/store-v1.yaml

@@ -0,0 +1,90 @@
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-v1.json"
+title: Store
+description: |
+  Experimental JSON representation of a Nix [Store](@docroot@/store/index.md).
+
+  This schema describes the JSON serialization of a Nix store.
+  We use it for (de)serializing in-memory "dummy stores" used for testing, but in principle the data represented in this schema could live in any type of store.
+
+  > **Warning**
+  >
+  > This JSON format is currently
+  > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
+  > and subject to change.
+
+type: object
+required:
+  - config
+  - contents
+  - derivations
+  - buildTrace
+properties:
+  config:
+    "$ref": "#/$defs/storeConfig"
+
+  contents:
+    type: object
+    title: Store Objects
+    description: |
+      Map of [store path](@docroot@/store/store-path.md) base names to [store objects](@docroot@/store/store-object.md).
+    patternProperties:
+      "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$":
+        type: object
+        title: Store Object
+        required:
+          - info
+          - contents
+        properties:
+          info:
+            "$ref": "./store-object-info-v2.yaml#/$defs/impure"
+            title: Store Object Info
+            description: |
+              Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc.
+          contents:
+            "$ref": "./file-system-object-v1.yaml"
+            title: File System Object Contents
+            description: |
+              The actual [file system object](@docroot@/store/file-system-object.md) contents of this store path.
+        additionalProperties: false
+    additionalProperties: false
+
+  derivations:
+    type: object
+    title: Derivations
+    description: |
+      Map of [store path](@docroot@/store/store-path.md) base names (always ending in `.drv`) to [derivations](@docroot@/store/derivation/index.md).
+    patternProperties:
+      "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$":
+        "$ref": "./derivation-v4.yaml"
+    additionalProperties: false
+
+  buildTrace:
+    type: object
+    title: Build Trace
+    description: |
+      Map of output hashes (base64 SHA256) to maps of output names to realisations.
+      Records which outputs have been built and their realisations.
+      See [Build Trace](@docroot@/store/build-trace.md) for more details.
+    patternProperties:
+      "^[A-Za-z0-9+/]{43}=$":
+        type: object
+        additionalProperties:
+          "$ref": "./build-trace-entry-v1.yaml#/$defs/value"
+    additionalProperties: false
+
+"$defs":
+  storeConfig:
+    title: Store Configuration
+    description: |
+      Configuration for the store, including the store directory path.
+    type: object
+    required:
+      - store
+    properties:
+      store:
+        type: string
+        title: Store Directory
+        description: |
+          The store directory path (e.g., `/nix/store`).
+    additionalProperties: false

doc/manual/source/protocols/json/store.md

@@ -0,0 +1,21 @@
+{{#include store-v1-fixed.md}}
+
+## Examples
+
+### Empty store
+
+```json
+{{#include schema/store-v1/empty.json}}
+```
+
+### Store with one file
+
+```json
+{{#include schema/store-v1/one-flat-file.json}}
+```
+
+### Store with one derivation
+
+```json
+{{#include schema/store-v1/one-derivation.json}}
+```

src/json-schema-checks/meson.build

@@ -200,6 +200,19 @@ schemas += [
   },
 ]
 
+# Dummy store
+schemas += [
+  {
+    'stem' : 'store',
+    'schema' : schema_dir / 'store-v1.yaml',
+    'files' : [
+      'empty.json',
+      'one-flat-file.json',
+      'one-derivation.json',
+    ],
+  },
+]
+
 # Validate each example against the schema
 foreach schema : schemas
   stem = schema['stem']

src/json-schema-checks/package.nix

@@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: {
     ../../src/libstore-tests/data/path-info
     ../../src/libstore-tests/data/nar-info
     ../../src/libstore-tests/data/build-result
+    ../../src/libstore-tests/data/dummy-store
     ./.
   ];
 

src/json-schema-checks/store

@@ -0,0 +1 @@
+../../src/libstore-tests/data/dummy-store

src/libstore-tests/data/dummy-store/empty.json

@@ -0,0 +1,8 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {}
+}

src/libstore-tests/data/dummy-store/one-derivation.json

@@ -0,0 +1,22 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {
+    "rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "foo",
+      "outputs": {},
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/dummy-store/one-flat-file.json

@@ -0,0 +1,38 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": {
+      "contents": {
+        "contents": "asdf",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": {
+            "algorithm": "sha256",
+            "format": "base64",
+            "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
+          },
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": {
+          "algorithm": "sha256",
+          "format": "base64",
+          "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
+        },
+        "narSize": 120,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/dummy-store.cc

@@ -1,11 +1,32 @@
 #include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
 
+#include "nix/util/memory-source-accessor.hh"
 #include "nix/store/dummy-store-impl.hh"
 #include "nix/store/globals.hh"
 #include "nix/store/realisation.hh"
 
+#include "nix/util/tests/json-characterization.hh"
+
 namespace nix {
 
+class DummyStoreTest : public virtual CharacterizationTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "dummy-store";
+
+public:
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+
+    static void SetUpTestSuite()
+    {
+        initLibStore(false);
+    }
+};
+
 TEST(DummyStore, realisation_read)
 {
     initLibStore(/*loadConfig=*/false);
@@ -35,4 +56,76 @@ TEST(DummyStore, realisation_read)
     EXPECT_EQ(*value2, value);
 }
 
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+using nlohmann::json;
+
+struct DummyStoreJsonTest : DummyStoreTest,
+                            JsonCharacterizationTest<ref<DummyStore>>,
+                            ::testing::WithParamInterface<std::pair<std::string_view, ref<DummyStore>>>
+{};
+
+TEST_P(DummyStoreJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    using namespace nlohmann;
+    /* Cannot use `readJsonTest` because need to dereference the stores
+       for equality. */
+    readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
+        auto encoded = json::parse(encodedRaw);
+        ref<DummyStore> decoded = adl_serializer<ref<DummyStore>>::from_json(encoded);
+        ASSERT_EQ(*decoded, *expected);
+    });
+}
+
+TEST_P(DummyStoreJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] {
+    initLibStore(false);
+    auto writeCfg = make_ref<DummyStore::Config>(DummyStore::Config::Params{});
+    writeCfg->readOnly = false;
+    return ::testing::Values(
+        std::pair{
+            "empty",
+            make_ref<DummyStore::Config>(DummyStore::Config::Params{})->openDummyStore(),
+        },
+        std::pair{
+            "one-flat-file",
+            [&] {
+                auto store = writeCfg->openDummyStore();
+                store->addToStore(
+                    "my-file",
+                    SourcePath{
+                        [] {
+                            auto sc = make_ref<MemorySourceAccessor>();
+                            sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                                .executable = false,
+                                .contents = "asdf",
+                            }};
+                            return sc;
+                        }(),
+                    },
+                    ContentAddressMethod::Raw::NixArchive,
+                    HashAlgorithm::SHA256);
+                return store;
+            }(),
+        },
+        std::pair{
+            "one-derivation",
+            [&] {
+                auto store = writeCfg->openDummyStore();
+                Derivation drv;
+                drv.name = "foo";
+                store->writeDerivation(drv);
+                return store;
+            }(),
+        });
+}());
+
 } // namespace nix

src/libstore/dummy-store.cc

@@ -2,6 +2,7 @@
 #include "nix/util/archive.hh"
 #include "nix/util/callback.hh"
 #include "nix/util/memory-source-accessor.hh"
+#include "nix/util/json-utils.hh"
 #include "nix/store/dummy-store-impl.hh"
 #include "nix/store/realisation.hh"
 
@@ -16,6 +17,16 @@ std::string DummyStoreConfig::doc()
         ;
 }
 
+bool DummyStore::PathInfoAndContents::operator==(const PathInfoAndContents & other) const
+{
+    return info == other.info && contents->root == other.contents->root;
+}
+
+bool DummyStore::operator==(const DummyStore & other) const
+{
+    return contents == other.contents && buildTrace == other.buildTrace;
+}
+
 namespace {
 
 class WholeStoreViewAccessor : public SourceAccessor
@@ -377,3 +388,100 @@ ref<DummyStore> DummyStore::Config::openDummyStore() const
 static RegisterStoreImplementation<DummyStore::Config> regDummyStore;
 
 } // namespace nix
+
+namespace nlohmann {
+
+using namespace nix;
+
+DummyStore::PathInfoAndContents adl_serializer<DummyStore::PathInfoAndContents>::from_json(const json & json)
+{
+    auto & obj = getObject(json);
+    return DummyStore::PathInfoAndContents{
+        .info = valueAt(obj, "info"),
+        .contents = make_ref<MemorySourceAccessor>(valueAt(obj, "contents")),
+    };
+}
+
+void adl_serializer<DummyStore::PathInfoAndContents>::to_json(json & json, const DummyStore::PathInfoAndContents & val)
+{
+    json = {
+        {"info", val.info},
+        {"contents", *val.contents},
+    };
+}
+
+ref<DummyStoreConfig> adl_serializer<ref<DummyStore::Config>>::from_json(const json & json)
+{
+    auto & obj = getObject(json);
+    auto cfg = make_ref<DummyStore::Config>(DummyStore::Config::Params{});
+    const_cast<PathSetting &>(cfg->storeDir_).set(getString(valueAt(obj, "store")));
+    cfg->readOnly = true;
+    return cfg;
+}
+
+void adl_serializer<DummyStoreConfig>::to_json(json & json, const DummyStoreConfig & val)
+{
+    json = {
+        {"store", val.storeDir},
+    };
+}
+
+ref<DummyStore> adl_serializer<ref<DummyStore>>::from_json(const json & json)
+{
+    auto & obj = getObject(json);
+    ref<DummyStore> res = adl_serializer<ref<DummyStoreConfig>>::from_json(valueAt(obj, "config"))->openDummyStore();
+    for (auto & [k, v] : getObject(valueAt(obj, "contents")))
+        res->contents.insert({StorePath{k}, v});
+    for (auto & [k, v] : getObject(valueAt(obj, "derivations")))
+        res->derivations.insert({StorePath{k}, v});
+    for (auto & [k0, v] : getObject(valueAt(obj, "buildTrace"))) {
+        for (auto & [k1, v2] : getObject(v)) {
+            auto vref = make_ref<UnkeyedRealisation>(v2);
+            res->buildTrace.insert_or_visit(
+                {
+                    Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64),
+                    {{k1, vref}},
+                },
+                [&](auto & kv) { kv.second.insert_or_assign(k1, vref); });
+        }
+    }
+    return res;
+}
+
+void adl_serializer<DummyStore>::to_json(json & json, const DummyStore & val)
+{
+    json = {
+        {"config", *val.config},
+        {"contents",
+         [&] {
+             auto obj = json::object();
+             val.contents.cvisit_all([&](const auto & kv) {
+                 auto & [k, v] = kv;
+                 obj[k.to_string()] = v;
+             });
+             return obj;
+         }()},
+        {"derivations",
+         [&] {
+             auto obj = json::object();
+             val.derivations.cvisit_all([&](const auto & kv) {
+                 auto & [k, v] = kv;
+                 obj[k.to_string()] = v;
+             });
+             return obj;
+         }()},
+        {"buildTrace",
+         [&] {
+             auto obj = json::object();
+             val.buildTrace.cvisit_all([&](const auto & kv) {
+                 auto & [k, v] = kv;
+                 auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object();
+                 for (auto & [k2, v2] : kv.second)
+                     obj2[k2] = *v2;
+             });
+             return obj;
+         }()},
+    };
+}
+
+} // namespace nlohmann

src/libstore/include/nix/store/dummy-store-impl.hh

@@ -23,6 +23,8 @@ struct DummyStore : virtual Store
     {
         UnkeyedValidPathInfo info;
         ref<MemorySourceAccessor> contents;
+
+        bool operator==(const PathInfoAndContents &) const;
     };
 
     /**
@@ -54,6 +56,14 @@ struct DummyStore : virtual Store
         , config(config)
     {
     }
+
+    bool operator==(const DummyStore &) const;
 };
 
+template<>
+struct json_avoids_null<DummyStore::PathInfoAndContents> : std::true_type
+{};
+
 } // namespace nix
+
+JSON_IMPL(nix::DummyStore::PathInfoAndContents)

src/libstore/include/nix/store/dummy-store.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "nix/store/store-api.hh"
+#include "nix/util/json-impls.hh"
 
 #include <boost/unordered/concurrent_flat_map.hpp>
 
@@ -65,4 +66,33 @@ struct DummyStoreConfig : public std::enable_shared_from_this<DummyStoreConfig>,
     }
 };
 
+template<>
+struct json_avoids_null<nix::DummyStoreConfig> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<ref<nix::DummyStoreConfig>> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<nix::DummyStore> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<ref<nix::DummyStore>> : std::true_type
+{};
+
 } // namespace nix
+
+namespace nlohmann {
+
+template<>
+JSON_IMPL_INNER_TO(nix::DummyStoreConfig);
+template<>
+JSON_IMPL_INNER_FROM(nix::ref<nix::DummyStoreConfig>);
+template<>
+JSON_IMPL_INNER_TO(nix::DummyStore);
+template<>
+JSON_IMPL_INNER_FROM(nix::ref<nix::DummyStore>);
+
+} // namespace nlohmann

src/libutil-test-support/include/nix/util/tests/json-characterization.hh

@@ -5,6 +5,7 @@
 #include <nlohmann/json.hpp>
 
 #include "nix/util/types.hh"
+#include "nix/util/ref.hh"
 #include "nix/util/file-system.hh"
 
 #include "nix/util/tests/characterization.hh"
@@ -39,6 +40,21 @@ void writeJsonTest(CharacterizationTest & test, PathView testStem, const T & val
         [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });
 }
 
+/**
+ * Specialization for when we need to return do JSON -> `ref<T>`, but
+ * `const T &` -> JSON.
+ */
+template<typename T>
+void writeJsonTest(CharacterizationTest & test, PathView testStem, const ref<T> & value)
+{
+    using namespace nlohmann;
+    test.writeTest(
+        Path{testStem} + ".json",
+        [&]() -> json { return static_cast<json>(*value); },
+        [](const auto & file) { return json::parse(readFile(file)); },
+        [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });
+}
+
 /**
  * Mixin class for writing characterization tests for `nlohmann::json`
  * conversions for a given type.

src/libutil/include/nix/util/json-impls.hh

@@ -6,18 +6,30 @@
 #include "nix/util/experimental-features.hh"
 
 // Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
+#define JSON_IMPL_INNER_TO(TYPE)                          \
+    struct adl_serializer<TYPE>                           \
+    {                                                     \
+        static void to_json(json & json, const TYPE & t); \
+    }
+
+#define JSON_IMPL_INNER_FROM(TYPE)                \
+    struct adl_serializer<TYPE>                   \
+    {                                             \
+        static TYPE from_json(const json & json); \
+    }
+
 #define JSON_IMPL_INNER(TYPE)                             \
     struct adl_serializer<TYPE>                           \
     {                                                     \
         static TYPE from_json(const json & json);         \
         static void to_json(json & json, const TYPE & t); \
-    };
+    }
 
 #define JSON_IMPL(TYPE)    \
     namespace nlohmann {   \
     using namespace nix;   \
     template<>             \
-    JSON_IMPL_INNER(TYPE) \
+    JSON_IMPL_INNER(TYPE); \
     }
 
 #define JSON_IMPL_WITH_XP_FEATURES(TYPE)                                                                            \

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -188,23 +188,23 @@ using namespace nix;
 
 #define ARG fso::Regular<RegularContents>
 template<typename RegularContents>
-JSON_IMPL_INNER(ARG)
+JSON_IMPL_INNER(ARG);
 #undef ARG
 
 #define ARG fso::DirectoryT<Child>
 template<typename Child>
-JSON_IMPL_INNER(ARG)
+JSON_IMPL_INNER(ARG);
 #undef ARG
 
 template<>
-JSON_IMPL_INNER(fso::Symlink)
+JSON_IMPL_INNER(fso::Symlink);
 
 template<>
-JSON_IMPL_INNER(fso::Opaque)
+JSON_IMPL_INNER(fso::Opaque);
 
 #define ARG fso::VariantT<RegularContents, recur>
 template<typename RegularContents, bool recur>
-JSON_IMPL_INNER(ARG)
+JSON_IMPL_INNER(ARG);
 #undef ARG
 
 } // namespace nlohmann