1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-22 18:29:36 +01:00

JSON impl and Schema for DummyStore

This is the "keystone" that puts most of the other store-layer JSON
formats together.
This commit is contained in:
John Ericson 2025-09-28 02:15:20 -04:00
parent 152e7e48c1
commit 8ffc6b5f5e
20 changed files with 559 additions and 66 deletions

View file

@ -46,6 +46,7 @@ mkMesonDerivation (finalAttrs: {
../../src/libstore-tests/data/path-info
../../src/libstore-tests/data/nar-info
../../src/libstore-tests/data/build-result
../../src/libstore-tests/data/dummy-store
# Too many different types of files to filter for now
../../doc/manual
./.

View file

@ -130,6 +130,7 @@
- [Deriving Path](protocols/json/deriving-path.md)
- [Build Trace Entry](protocols/json/build-trace-entry.md)
- [Build Result](protocols/json/build-result.md)
- [Store](protocols/json/store.md)
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
- [Store Path Specification](protocols/store-path.md)
- [Nix Archive (NAR) Format](protocols/nix-archive/index.md)

View file

@ -18,6 +18,7 @@ schemas = [
'deriving-path-v1',
'build-trace-entry-v1',
'build-result-v1',
'store-v1',
]
schema_files = files()

View file

@ -4,71 +4,97 @@ title: Build Trace Entry
description: |
A record of a successful build outcome for a specific derivation output.
This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md) entry.
This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md).
> **Warning**
>
> This JSON format is currently
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
> and subject to change.
type: object
required:
- id
- outPath
- dependentRealisations
- signatures
allOf:
- "$ref": "#/$defs/key"
- "$ref": "#/$defs/value"
properties:
id:
type: string
title: Derivation Output ID
pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$"
description: |
Unique identifier for the derivation output that was built.
Format: `{hash-quotient-drv}!{output-name}`
- **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv).
Begins with `sha256:`.
- **output-name**: Name of the specific output (e.g., "out", "dev", "doc")
Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"`
outPath:
"$ref": "store-path-v1.yaml"
title: Output Store Path
description: |
The path to the store object that resulted from building this derivation for the given output name.
dependentRealisations:
type: object
title: Underlying Base Build Trace
description: |
This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
Keys are derivation output IDs (same format as the main `id` field).
Values are the store paths that those dependencies resolved to.
As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
This is the set of base build trace entries that this derived build trace is derived from.
(The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
patternProperties:
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
$ref: "store-path-v1.yaml"
title: Dependent Store Path
description: Store path that this dependency resolved to during the build
additionalProperties: false
signatures:
type: array
title: Build Signatures
description: |
A set of cryptographic signatures attesting to the authenticity of this build trace entry.
items:
type: string
title: Signature
description: A single cryptographic signature
id: {}
outPath: {}
dependentRealisations: {}
signatures: {}
additionalProperties: false
"$defs":
key:
title: Build Trace Key
description: |
A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair.
This is the "key" part, refering to a derivation and output.
type: object
required:
- id
properties:
id:
type: string
title: Derivation Output ID
pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$"
description: |
Unique identifier for the derivation output that was built.
Format: `{hash-quotient-drv}!{output-name}`
- **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv).
Begins with `sha256:`.
- **output-name**: Name of the specific output (e.g., "out", "dev", "doc")
Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"`
value:
title: Build Trace Value
description: |
A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair.
This is the "value" part, describing an output.
type: object
required:
- outPath
- dependentRealisations
- signatures
properties:
outPath:
"$ref": "store-path-v1.yaml"
title: Output Store Path
description: |
The path to the store object that resulted from building this derivation for the given output name.
dependentRealisations:
type: object
title: Underlying Base Build Trace
description: |
This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
Keys are derivation output IDs (same format as the main `id` field).
Values are the store paths that those dependencies resolved to.
As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
This is the set of base build trace entries that this derived build trace is derived from.
(The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
patternProperties:
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
"$ref": "store-path-v1.yaml"
title: Dependent Store Path
description: Store path that this dependency resolved to during the build
additionalProperties: false
signatures:
type: array
title: Build Signatures
description: |
A set of cryptographic signatures attesting to the authenticity of this build trace entry.
items:
type: string
title: Signature
description: A single cryptographic signature

View file

@ -0,0 +1 @@
../../../../../../src/libstore-tests/data/dummy-store

View file

@ -0,0 +1,90 @@
"$schema": "http://json-schema.org/draft-04/schema"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-v1.json"
title: Store
description: |
Experimental JSON representation of a Nix [Store](@docroot@/store/index.md).
This schema describes the JSON serialization of a Nix store.
We use it for (de)serializing in-memory "dummy stores" used for testing, but in principle the data represented in this schema could live in any type of store.
> **Warning**
>
> This JSON format is currently
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
> and subject to change.
type: object
required:
- config
- contents
- derivations
- buildTrace
properties:
config:
"$ref": "#/$defs/storeConfig"
contents:
type: object
title: Store Objects
description: |
Map of [store path](@docroot@/store/store-path.md) base names to [store objects](@docroot@/store/store-object.md).
patternProperties:
"^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$":
type: object
title: Store Object
required:
- info
- contents
properties:
info:
"$ref": "./store-object-info-v2.yaml#/$defs/impure"
title: Store Object Info
description: |
Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc.
contents:
"$ref": "./file-system-object-v1.yaml"
title: File System Object Contents
description: |
The actual [file system object](@docroot@/store/file-system-object.md) contents of this store path.
additionalProperties: false
additionalProperties: false
derivations:
type: object
title: Derivations
description: |
Map of [store path](@docroot@/store/store-path.md) base names (always ending in `.drv`) to [derivations](@docroot@/store/derivation/index.md).
patternProperties:
"^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$":
"$ref": "./derivation-v4.yaml"
additionalProperties: false
buildTrace:
type: object
title: Build Trace
description: |
Map of output hashes (base64 SHA256) to maps of output names to realisations.
Records which outputs have been built and their realisations.
See [Build Trace](@docroot@/store/build-trace.md) for more details.
patternProperties:
"^[A-Za-z0-9+/]{43}=$":
type: object
additionalProperties:
"$ref": "./build-trace-entry-v1.yaml#/$defs/value"
additionalProperties: false
"$defs":
storeConfig:
title: Store Configuration
description: |
Configuration for the store, including the store directory path.
type: object
required:
- store
properties:
store:
type: string
title: Store Directory
description: |
The store directory path (e.g., `/nix/store`).
additionalProperties: false

View file

@ -0,0 +1,21 @@
{{#include store-v1-fixed.md}}
## Examples
### Empty store
```json
{{#include schema/store-v1/empty.json}}
```
### Store with one file
```json
{{#include schema/store-v1/one-flat-file.json}}
```
### Store with one derivation
```json
{{#include schema/store-v1/one-derivation.json}}
```

View file

@ -200,6 +200,19 @@ schemas += [
},
]
# Dummy store
schemas += [
{
'stem' : 'store',
'schema' : schema_dir / 'store-v1.yaml',
'files' : [
'empty.json',
'one-flat-file.json',
'one-derivation.json',
],
},
]
# Validate each example against the schema
foreach schema : schemas
stem = schema['stem']

View file

@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: {
../../src/libstore-tests/data/path-info
../../src/libstore-tests/data/nar-info
../../src/libstore-tests/data/build-result
../../src/libstore-tests/data/dummy-store
./.
];

View file

@ -0,0 +1 @@
../../src/libstore-tests/data/dummy-store

View file

@ -0,0 +1,8 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {},
"derivations": {}
}

View file

@ -0,0 +1,22 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {},
"derivations": {
"rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "foo",
"outputs": {},
"system": "",
"version": 4
}
}
}

View file

@ -0,0 +1,38 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {
"5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": {
"contents": {
"contents": "asdf",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": {
"algorithm": "sha256",
"format": "base64",
"hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
},
"method": "nar"
},
"deriver": null,
"narHash": {
"algorithm": "sha256",
"format": "base64",
"hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
},
"narSize": 120,
"references": [],
"registrationTime": null,
"signatures": [],
"ultimate": false,
"version": 2
}
}
},
"derivations": {}
}

View file

@ -1,11 +1,32 @@
#include <gtest/gtest.h>
#include <nlohmann/json.hpp>
#include "nix/util/memory-source-accessor.hh"
#include "nix/store/dummy-store-impl.hh"
#include "nix/store/globals.hh"
#include "nix/store/realisation.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
class DummyStoreTest : public virtual CharacterizationTest
{
std::filesystem::path unitTestData = getUnitTestData() / "dummy-store";
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
static void SetUpTestSuite()
{
initLibStore(false);
}
};
TEST(DummyStore, realisation_read)
{
initLibStore(/*loadConfig=*/false);
@ -35,4 +56,76 @@ TEST(DummyStore, realisation_read)
EXPECT_EQ(*value2, value);
}
/* ----------------------------------------------------------------------------
* JSON
* --------------------------------------------------------------------------*/
using nlohmann::json;
struct DummyStoreJsonTest : DummyStoreTest,
JsonCharacterizationTest<ref<DummyStore>>,
::testing::WithParamInterface<std::pair<std::string_view, ref<DummyStore>>>
{};
TEST_P(DummyStoreJsonTest, from_json)
{
auto & [name, expected] = GetParam();
using namespace nlohmann;
/* Cannot use `readJsonTest` because need to dereference the stores
for equality. */
readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
auto encoded = json::parse(encodedRaw);
ref<DummyStore> decoded = adl_serializer<ref<DummyStore>>::from_json(encoded);
ASSERT_EQ(*decoded, *expected);
});
}
TEST_P(DummyStoreJsonTest, to_json)
{
auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] {
initLibStore(false);
auto writeCfg = make_ref<DummyStore::Config>(DummyStore::Config::Params{});
writeCfg->readOnly = false;
return ::testing::Values(
std::pair{
"empty",
make_ref<DummyStore::Config>(DummyStore::Config::Params{})->openDummyStore(),
},
std::pair{
"one-flat-file",
[&] {
auto store = writeCfg->openDummyStore();
store->addToStore(
"my-file",
SourcePath{
[] {
auto sc = make_ref<MemorySourceAccessor>();
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents = "asdf",
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256);
return store;
}(),
},
std::pair{
"one-derivation",
[&] {
auto store = writeCfg->openDummyStore();
Derivation drv;
drv.name = "foo";
store->writeDerivation(drv);
return store;
}(),
});
}());
} // namespace nix

View file

@ -2,6 +2,7 @@
#include "nix/util/archive.hh"
#include "nix/util/callback.hh"
#include "nix/util/memory-source-accessor.hh"
#include "nix/util/json-utils.hh"
#include "nix/store/dummy-store-impl.hh"
#include "nix/store/realisation.hh"
@ -16,6 +17,16 @@ std::string DummyStoreConfig::doc()
;
}
bool DummyStore::PathInfoAndContents::operator==(const PathInfoAndContents & other) const
{
return info == other.info && contents->root == other.contents->root;
}
bool DummyStore::operator==(const DummyStore & other) const
{
return contents == other.contents && buildTrace == other.buildTrace;
}
namespace {
class WholeStoreViewAccessor : public SourceAccessor
@ -377,3 +388,100 @@ ref<DummyStore> DummyStore::Config::openDummyStore() const
static RegisterStoreImplementation<DummyStore::Config> regDummyStore;
} // namespace nix
namespace nlohmann {
using namespace nix;
DummyStore::PathInfoAndContents adl_serializer<DummyStore::PathInfoAndContents>::from_json(const json & json)
{
auto & obj = getObject(json);
return DummyStore::PathInfoAndContents{
.info = valueAt(obj, "info"),
.contents = make_ref<MemorySourceAccessor>(valueAt(obj, "contents")),
};
}
void adl_serializer<DummyStore::PathInfoAndContents>::to_json(json & json, const DummyStore::PathInfoAndContents & val)
{
json = {
{"info", val.info},
{"contents", *val.contents},
};
}
ref<DummyStoreConfig> adl_serializer<ref<DummyStore::Config>>::from_json(const json & json)
{
auto & obj = getObject(json);
auto cfg = make_ref<DummyStore::Config>(DummyStore::Config::Params{});
const_cast<PathSetting &>(cfg->storeDir_).set(getString(valueAt(obj, "store")));
cfg->readOnly = true;
return cfg;
}
void adl_serializer<DummyStoreConfig>::to_json(json & json, const DummyStoreConfig & val)
{
json = {
{"store", val.storeDir},
};
}
ref<DummyStore> adl_serializer<ref<DummyStore>>::from_json(const json & json)
{
auto & obj = getObject(json);
ref<DummyStore> res = adl_serializer<ref<DummyStoreConfig>>::from_json(valueAt(obj, "config"))->openDummyStore();
for (auto & [k, v] : getObject(valueAt(obj, "contents")))
res->contents.insert({StorePath{k}, v});
for (auto & [k, v] : getObject(valueAt(obj, "derivations")))
res->derivations.insert({StorePath{k}, v});
for (auto & [k0, v] : getObject(valueAt(obj, "buildTrace"))) {
for (auto & [k1, v2] : getObject(v)) {
auto vref = make_ref<UnkeyedRealisation>(v2);
res->buildTrace.insert_or_visit(
{
Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64),
{{k1, vref}},
},
[&](auto & kv) { kv.second.insert_or_assign(k1, vref); });
}
}
return res;
}
void adl_serializer<DummyStore>::to_json(json & json, const DummyStore & val)
{
json = {
{"config", *val.config},
{"contents",
[&] {
auto obj = json::object();
val.contents.cvisit_all([&](const auto & kv) {
auto & [k, v] = kv;
obj[k.to_string()] = v;
});
return obj;
}()},
{"derivations",
[&] {
auto obj = json::object();
val.derivations.cvisit_all([&](const auto & kv) {
auto & [k, v] = kv;
obj[k.to_string()] = v;
});
return obj;
}()},
{"buildTrace",
[&] {
auto obj = json::object();
val.buildTrace.cvisit_all([&](const auto & kv) {
auto & [k, v] = kv;
auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object();
for (auto & [k2, v2] : kv.second)
obj2[k2] = *v2;
});
return obj;
}()},
};
}
} // namespace nlohmann

View file

@ -23,6 +23,8 @@ struct DummyStore : virtual Store
{
UnkeyedValidPathInfo info;
ref<MemorySourceAccessor> contents;
bool operator==(const PathInfoAndContents &) const;
};
/**
@ -54,6 +56,14 @@ struct DummyStore : virtual Store
, config(config)
{
}
bool operator==(const DummyStore &) const;
};
template<>
struct json_avoids_null<DummyStore::PathInfoAndContents> : std::true_type
{};
} // namespace nix
JSON_IMPL(nix::DummyStore::PathInfoAndContents)

View file

@ -2,6 +2,7 @@
///@file
#include "nix/store/store-api.hh"
#include "nix/util/json-impls.hh"
#include <boost/unordered/concurrent_flat_map.hpp>
@ -65,4 +66,33 @@ struct DummyStoreConfig : public std::enable_shared_from_this<DummyStoreConfig>,
}
};
template<>
struct json_avoids_null<nix::DummyStoreConfig> : std::true_type
{};
template<>
struct json_avoids_null<ref<nix::DummyStoreConfig>> : std::true_type
{};
template<>
struct json_avoids_null<nix::DummyStore> : std::true_type
{};
template<>
struct json_avoids_null<ref<nix::DummyStore>> : std::true_type
{};
} // namespace nix
namespace nlohmann {
template<>
JSON_IMPL_INNER_TO(nix::DummyStoreConfig);
template<>
JSON_IMPL_INNER_FROM(nix::ref<nix::DummyStoreConfig>);
template<>
JSON_IMPL_INNER_TO(nix::DummyStore);
template<>
JSON_IMPL_INNER_FROM(nix::ref<nix::DummyStore>);
} // namespace nlohmann

View file

@ -5,6 +5,7 @@
#include <nlohmann/json.hpp>
#include "nix/util/types.hh"
#include "nix/util/ref.hh"
#include "nix/util/file-system.hh"
#include "nix/util/tests/characterization.hh"
@ -39,6 +40,21 @@ void writeJsonTest(CharacterizationTest & test, PathView testStem, const T & val
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });
}
/**
* Specialization for when we need to return do JSON -> `ref<T>`, but
* `const T &` -> JSON.
*/
template<typename T>
void writeJsonTest(CharacterizationTest & test, PathView testStem, const ref<T> & value)
{
using namespace nlohmann;
test.writeTest(
Path{testStem} + ".json",
[&]() -> json { return static_cast<json>(*value); },
[](const auto & file) { return json::parse(readFile(file)); },
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });
}
/**
* Mixin class for writing characterization tests for `nlohmann::json`
* conversions for a given type.

View file

@ -6,18 +6,30 @@
#include "nix/util/experimental-features.hh"
// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
#define JSON_IMPL_INNER_TO(TYPE) \
struct adl_serializer<TYPE> \
{ \
static void to_json(json & json, const TYPE & t); \
}
#define JSON_IMPL_INNER_FROM(TYPE) \
struct adl_serializer<TYPE> \
{ \
static TYPE from_json(const json & json); \
}
#define JSON_IMPL_INNER(TYPE) \
struct adl_serializer<TYPE> \
{ \
static TYPE from_json(const json & json); \
static void to_json(json & json, const TYPE & t); \
};
}
#define JSON_IMPL(TYPE) \
namespace nlohmann { \
using namespace nix; \
template<> \
JSON_IMPL_INNER(TYPE) \
#define JSON_IMPL(TYPE) \
namespace nlohmann { \
using namespace nix; \
template<> \
JSON_IMPL_INNER(TYPE); \
}
#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \

View file

@ -188,23 +188,23 @@ using namespace nix;
#define ARG fso::Regular<RegularContents>
template<typename RegularContents>
JSON_IMPL_INNER(ARG)
JSON_IMPL_INNER(ARG);
#undef ARG
#define ARG fso::DirectoryT<Child>
template<typename Child>
JSON_IMPL_INNER(ARG)
JSON_IMPL_INNER(ARG);
#undef ARG
template<>
JSON_IMPL_INNER(fso::Symlink)
JSON_IMPL_INNER(fso::Symlink);
template<>
JSON_IMPL_INNER(fso::Opaque)
JSON_IMPL_INNER(fso::Opaque);
#define ARG fso::VariantT<RegularContents, recur>
template<typename RegularContents, bool recur>
JSON_IMPL_INNER(ARG)
JSON_IMPL_INNER(ARG);
#undef ARG
} // namespace nlohmann