diff --git a/doc/manual/package.nix b/doc/manual/package.nix index b78ccee88..1d7d73ce9 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -47,6 +47,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info ../../src/libstore-tests/data/build-result + ../../src/libstore-tests/data/dummy-store # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 64bf593f4..a8e286314 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -131,6 +131,7 @@ - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) - [Build Result](protocols/json/build-result.md) + - [Store](protocols/json/store.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 8780d8057..ab9d76d3e 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -19,6 +19,7 @@ schemas = [ 'deriving-path-v1', 'build-trace-entry-v1', 'build-result-v1', + 'store-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml index cabf2c350..a85738b50 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml @@ -4,71 +4,97 @@ title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. - This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md) entry. + This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md). > **Warning** > > This JSON format is currently > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) > and subject to change. - -type: object required: - id - outPath - dependentRealisations - signatures +allOf: + - "$ref": "#/$defs/key" + - "$ref": "#/$defs/value" properties: - id: - type: string - title: Derivation Output ID - pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" - description: | - Unique identifier for the derivation output that was built. - - Format: `{hash-quotient-drv}!{output-name}` - - - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). - Begins with `sha256:`. - - - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") - - Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` - - outPath: - "$ref": "store-path-v1.yaml" - title: Output Store Path - description: | - The path to the store object that resulted from building this derivation for the given output name. - - dependentRealisations: - type: object - title: Underlying Base Build Trace - description: | - This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence. - - Keys are derivation output IDs (same format as the main `id` field). - Values are the store paths that those dependencies resolved to. - - As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries. - This is the set of base build trace entries that this derived build trace is derived from. - (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.) - - patternProperties: - "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": - $ref: "store-path-v1.yaml" - title: Dependent Store Path - description: Store path that this dependency resolved to during the build - additionalProperties: false - - signatures: - type: array - title: Build Signatures - description: | - A set of cryptographic signatures attesting to the authenticity of this build trace entry. - items: - type: string - title: Signature - description: A single cryptographic signature - + id: {} + outPath: {} + dependentRealisations: {} + signatures: {} additionalProperties: false + +"$defs": + key: + title: Build Trace Key + description: | + A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + This is the "key" part, refering to a derivation and output. + type: object + required: + - id + properties: + id: + type: string + title: Derivation Output ID + pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" + description: | + Unique identifier for the derivation output that was built. + + Format: `{hash-quotient-drv}!{output-name}` + + - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). + Begins with `sha256:`. + + - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") + + Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` + + value: + title: Build Trace Value + description: | + A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + This is the "value" part, describing an output. + type: object + required: + - outPath + - dependentRealisations + - signatures + properties: + outPath: + "$ref": "store-path-v1.yaml" + title: Output Store Path + description: | + The path to the store object that resulted from building this derivation for the given output name. + + dependentRealisations: + type: object + title: Underlying Base Build Trace + description: | + This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence. + + Keys are derivation output IDs (same format as the main `id` field). + Values are the store paths that those dependencies resolved to. + + As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries. + This is the set of base build trace entries that this derived build trace is derived from. + (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.) + + patternProperties: + "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": + "$ref": "store-path-v1.yaml" + title: Dependent Store Path + description: Store path that this dependency resolved to during the build + additionalProperties: false + + signatures: + type: array + title: Build Signatures + description: | + A set of cryptographic signatures attesting to the authenticity of this build trace entry. + items: + type: string + title: Signature + description: A single cryptographic signature diff --git a/doc/manual/source/protocols/json/schema/store-v1 b/doc/manual/source/protocols/json/schema/store-v1 new file mode 120000 index 000000000..0cb61f962 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/dummy-store \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-v1.yaml b/doc/manual/source/protocols/json/schema/store-v1.yaml new file mode 100644 index 000000000..e0c6f8fed --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-v1.yaml @@ -0,0 +1,90 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-v1.json" +title: Store +description: | + Experimental JSON representation of a Nix [Store](@docroot@/store/index.md). + + This schema describes the JSON serialization of a Nix store. + We use it for (de)serializing in-memory "dummy stores" used for testing, but in principle the data represented in this schema could live in any type of store. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + +type: object +required: + - config + - contents + - derivations + - buildTrace +properties: + config: + "$ref": "#/$defs/storeConfig" + + contents: + type: object + title: Store Objects + description: | + Map of [store path](@docroot@/store/store-path.md) base names to [store objects](@docroot@/store/store-object.md). + patternProperties: + "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$": + type: object + title: Store Object + required: + - info + - contents + properties: + info: + "$ref": "./store-object-info-v2.yaml#/$defs/impure" + title: Store Object Info + description: | + Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc. + contents: + "$ref": "./file-system-object-v1.yaml" + title: File System Object Contents + description: | + The actual [file system object](@docroot@/store/file-system-object.md) contents of this store path. + additionalProperties: false + additionalProperties: false + + derivations: + type: object + title: Derivations + description: | + Map of [store path](@docroot@/store/store-path.md) base names (always ending in `.drv`) to [derivations](@docroot@/store/derivation/index.md). + patternProperties: + "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": + "$ref": "./derivation-v4.yaml" + additionalProperties: false + + buildTrace: + type: object + title: Build Trace + description: | + Map of output hashes (base64 SHA256) to maps of output names to realisations. + Records which outputs have been built and their realisations. + See [Build Trace](@docroot@/store/build-trace.md) for more details. + patternProperties: + "^[A-Za-z0-9+/]{43}=$": + type: object + additionalProperties: + "$ref": "./build-trace-entry-v1.yaml#/$defs/value" + additionalProperties: false + +"$defs": + storeConfig: + title: Store Configuration + description: | + Configuration for the store, including the store directory path. + type: object + required: + - store + properties: + store: + type: string + title: Store Directory + description: | + The store directory path (e.g., `/nix/store`). + additionalProperties: false diff --git a/doc/manual/source/protocols/json/store.md b/doc/manual/source/protocols/json/store.md new file mode 100644 index 000000000..951c1759e --- /dev/null +++ b/doc/manual/source/protocols/json/store.md @@ -0,0 +1,21 @@ +{{#include store-v1-fixed.md}} + +## Examples + +### Empty store + +```json +{{#include schema/store-v1/empty.json}} +``` + +### Store with one file + +```json +{{#include schema/store-v1/one-flat-file.json}} +``` + +### Store with one derivation + +```json +{{#include schema/store-v1/one-derivation.json}} +``` diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 73be4a47d..09c8cd048 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -212,6 +212,19 @@ schemas += [ }, ] +# Dummy store +schemas += [ + { + 'stem' : 'store', + 'schema' : schema_dir / 'store-v1.yaml', + 'files' : [ + 'empty.json', + 'one-flat-file.json', + 'one-derivation.json', + ], + }, +] + # Validate each example against the schema foreach schema : schemas stem = schema['stem'] diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index d9ca880e5..a5ee1f059 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info ../../src/libstore-tests/data/build-result + ../../src/libstore-tests/data/dummy-store ./. ]; diff --git a/src/json-schema-checks/store b/src/json-schema-checks/store new file mode 120000 index 000000000..442f0749a --- /dev/null +++ b/src/json-schema-checks/store @@ -0,0 +1 @@ +../../src/libstore-tests/data/dummy-store \ No newline at end of file diff --git a/src/libstore-tests/data/dummy-store/empty.json b/src/libstore-tests/data/dummy-store/empty.json new file mode 100644 index 000000000..93bec5153 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/empty.json @@ -0,0 +1,8 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": {} +} diff --git a/src/libstore-tests/data/dummy-store/one-derivation.json b/src/libstore-tests/data/dummy-store/one-derivation.json new file mode 100644 index 000000000..a3e3391e6 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-derivation.json @@ -0,0 +1,22 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": { + "rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "foo", + "outputs": {}, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json new file mode 100644 index 000000000..d572b4c4f --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -0,0 +1,38 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": { + "5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": { + "contents": { + "contents": "asdf", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "method": "nar" + }, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "narSize": 120, + "references": [], + "registrationTime": null, + "signatures": [], + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/dummy-store/one-realisation.json b/src/libstore-tests/data/dummy-store/one-realisation.json new file mode 100644 index 000000000..b5c8b8c56 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-realisation.json @@ -0,0 +1,16 @@ +{ + "buildTrace": { + "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": { + "out": { + "dependentRealisations": {}, + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": {} +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index c87b8d773..4a12dcf78 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -1,11 +1,32 @@ #include +#include +#include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/globals.hh" #include "nix/store/realisation.hh" +#include "nix/util/tests/json-characterization.hh" + namespace nix { +class DummyStoreTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "dummy-store"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + static void SetUpTestSuite() + { + initLibStore(false); + } +}; + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); @@ -35,4 +56,96 @@ TEST(DummyStore, realisation_read) EXPECT_EQ(*value2, value); } +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +struct DummyStoreJsonTest : DummyStoreTest, + JsonCharacterizationTest>, + ::testing::WithParamInterface>> +{}; + +TEST_P(DummyStoreJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + using namespace nlohmann; + /* Cannot use `readJsonTest` because need to dereference the stores + for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + ref decoded = adl_serializer>::from_json(encoded); + ASSERT_EQ(*decoded, *expected); + }); +} + +TEST_P(DummyStoreJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { + initLibStore(false); + auto writeCfg = make_ref(DummyStore::Config::Params{}); + writeCfg->readOnly = false; + return ::testing::Values( + std::pair{ + "empty", + make_ref(DummyStore::Config::Params{})->openDummyStore(), + }, + std::pair{ + "one-flat-file", + [&] { + auto store = writeCfg->openDummyStore(); + store->addToStore( + "my-file", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "asdf", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + return store; + }(), + }, + std::pair{ + "one-derivation", + [&] { + auto store = writeCfg->openDummyStore(); + Derivation drv; + drv.name = "foo"; + store->writeDerivation(drv); + return store; + }(), + }, + std::pair{ + "one-realisation", + [&] { + auto store = writeCfg->openDummyStore(); + store->buildTrace.insert_or_assign( + Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + }); + return store; + }(), + }); +}()); + } // namespace nix diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 852c38b75..aa763a679 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -2,6 +2,7 @@ #include "nix/util/archive.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/json-utils.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/realisation.hh" @@ -386,3 +387,100 @@ ref DummyStore::Config::openDummyStore() const static RegisterStoreImplementation regDummyStore; } // namespace nix + +namespace nlohmann { + +using namespace nix; + +DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return DummyStore::PathInfoAndContents{ + .info = valueAt(obj, "info"), + .contents = make_ref(valueAt(obj, "contents")), + }; +} + +void adl_serializer::to_json(json & json, const DummyStore::PathInfoAndContents & val) +{ + json = { + {"info", val.info}, + {"contents", *val.contents}, + }; +} + +ref adl_serializer>::from_json(const json & json) +{ + auto & obj = getObject(json); + auto cfg = make_ref(DummyStore::Config::Params{}); + const_cast(cfg->storeDir_).set(getString(valueAt(obj, "store"))); + cfg->readOnly = true; + return cfg; +} + +void adl_serializer::to_json(json & json, const DummyStoreConfig & val) +{ + json = { + {"store", val.storeDir}, + }; +} + +ref adl_serializer>::from_json(const json & json) +{ + auto & obj = getObject(json); + ref res = adl_serializer>::from_json(valueAt(obj, "config"))->openDummyStore(); + for (auto & [k, v] : getObject(valueAt(obj, "contents"))) + res->contents.insert({StorePath{k}, v}); + for (auto & [k, v] : getObject(valueAt(obj, "derivations"))) + res->derivations.insert({StorePath{k}, v}); + for (auto & [k0, v] : getObject(valueAt(obj, "buildTrace"))) { + for (auto & [k1, v2] : getObject(v)) { + UnkeyedRealisation realisation = v2; + res->buildTrace.insert_or_visit( + { + Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + {{k1, realisation}}, + }, + [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); + } + } + return res; +} + +void adl_serializer::to_json(json & json, const DummyStore & val) +{ + json = { + {"config", *val.config}, + {"contents", + [&] { + auto obj = json::object(); + val.contents.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, + {"derivations", + [&] { + auto obj = json::object(); + val.derivations.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, + {"buildTrace", + [&] { + auto obj = json::object(); + val.buildTrace.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + for (auto & [k2, v2] : kv.second) + obj2[k2] = v2; + }); + return obj; + }()}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 9b078eeaa..ac7ab9c68 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -60,4 +60,10 @@ struct DummyStore : virtual Store bool operator==(const DummyStore &) const; }; +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::DummyStore::PathInfoAndContents) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index d371c4e51..febf351c9 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/json-impls.hh" #include @@ -65,4 +66,33 @@ struct DummyStoreConfig : public std::enable_shared_from_this, } }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null> : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null> : std::true_type +{}; + } // namespace nix + +namespace nlohmann { + +template<> +JSON_IMPL_INNER_TO(nix::DummyStoreConfig); +template<> +JSON_IMPL_INNER_FROM(nix::ref); +template<> +JSON_IMPL_INNER_TO(nix::DummyStore); +template<> +JSON_IMPL_INNER_FROM(nix::ref); + +} // namespace nlohmann