diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 30486869e..fc64cee86 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -34,6 +34,7 @@ mkMesonDerivation (finalAttrs: { (fileset.unions [ ../../.version # For example JSON + ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash # Too many different types of files to filter for now ../../doc/manual diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index f74ed7043..6cb816af6 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -117,6 +117,7 @@ - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) + - [File System Object](protocols/json/file-system-object.md) - [Hash](protocols/json/hash.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) diff --git a/doc/manual/source/protocols/json/file-system-object.md b/doc/manual/source/protocols/json/file-system-object.md new file mode 100644 index 000000000..6517d73ca --- /dev/null +++ b/doc/manual/source/protocols/json/file-system-object.md @@ -0,0 +1,21 @@ +{{#include file-system-object-v1-fixed.md}} + +## Examples + +### Simple + +```json +{{#include schema/file-system-object-v1/simple.json}} +``` + +### Complex + +```json +{{#include schema/file-system-object-v1/complex.json}} +``` + + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 126e666e9..397c2dc5d 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -11,4 +11,5 @@ s/\\`/`/g # # As we have more such relative links, more replacements of this nature # should appear below. +s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 44795599c..d5c560fd3 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -9,6 +9,7 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false) json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ + 'file-system-object-v1', 'hash-v1', 'derivation-v3', ] diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1 b/doc/manual/source/protocols/json/schema/file-system-object-v1 new file mode 120000 index 000000000..cbb21a10d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1 @@ -0,0 +1 @@ +../../../../../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml new file mode 100644 index 000000000..c7154b18d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml @@ -0,0 +1,65 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json +title: File System Object +description: | + This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md). + + The schema is recursive because file system objects contain other file system objects. +type: object +required: ["type"] +properties: + type: + type: string + enum: ["regular", "symlink", "directory"] + +# Enforce conditional structure based on `type` +anyOf: + - $ref: "#/$defs/regular" + required: ["type", "contents"] + + - $ref: "#/$defs/symlink" + required: ["type", "target"] + + - $ref: "#/$defs/directory" + required: ["type", "contents"] + +"$defs": + regular: + title: Regular File + required: ["contents"] + properties: + type: + const: "regular" + contents: + type: string + description: Base64-encoded file contents + executable: + type: boolean + description: Whether the file is executable. + default: false + additionalProperties: false + + symlink: + title: Symbolic Link + required: ["target"] + properties: + type: + const: "symlink" + target: + type: string + description: Target path of the symlink. + additionalProperties: false + + directory: + title: Directory + required: ["contents"] + properties: + type: + const: "directory" + contents: + type: object + description: | + Map of names to nested file system objects (for type=directory) + additionalProperties: + $ref: "#" + additionalProperties: false diff --git a/src/json-schema-checks/file-system-object b/src/json-schema-checks/file-system-object new file mode 120000 index 000000000..b26e030c9 --- /dev/null +++ b/src/json-schema-checks/file-system-object @@ -0,0 +1 @@ +../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index ebd6f6b2b..1ab7f0dc3 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema' # Get all example files schemas = [ + { + 'stem' : 'file-system-object', + 'schema' : schema_dir / 'file-system-object-v1.yaml', + 'files' : [ + 'simple.json', + 'complex.json', + ], + }, { 'stem' : 'hash', 'schema' : schema_dir / 'hash-v1.yaml', @@ -64,8 +72,6 @@ foreach schema : schemas stem + '-schema-valid', jv, args : [ - '--map', - './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml', 'http://json-schema.org/draft-04/schema', schema_file, ], diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 41458adb8..6eddbd5dd 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: { fileset = lib.fileset.unions [ ../../.version ../../doc/manual/source/protocols/json/schema + ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/derivation ./. diff --git a/src/libutil-tests/data/memory-source-accessor/complex.json b/src/libutil-tests/data/memory-source-accessor/complex.json new file mode 100644 index 000000000..082956768 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/complex.json @@ -0,0 +1,24 @@ +{ + "contents": { + "bar": { + "contents": { + "baz": { + "contents": "Z29vZCBkYXksCg==", + "executable": true, + "type": "regular" + }, + "quux": { + "target": "/over/there", + "type": "symlink" + } + }, + "type": "directory" + }, + "foo": { + "contents": "aGVsbG8K", + "executable": false, + "type": "regular" + } + }, + "type": "directory" +} diff --git a/src/libutil-tests/data/memory-source-accessor/simple.json b/src/libutil-tests/data/memory-source-accessor/simple.json new file mode 100644 index 000000000..8c61fa730 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/simple.json @@ -0,0 +1,5 @@ +{ + "contents": "YXNkZg==", + "executable": false, + "type": "regular" +} diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index 6180a4cfc..f761c4433 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -224,42 +224,15 @@ TEST_F(GitTest, tree_sha256_write) }); } +namespace memory_source_accessor { + +extern ref exampleComplex(); + +} + TEST_F(GitTest, both_roundrip) { - using File = MemorySourceAccessor::File; - - auto files = make_ref(); - files->root = File::Directory{ - .contents{ - { - "foo", - File::Regular{ - .contents = "hello\n\0\n\tworld!", - }, - }, - { - "bar", - File::Directory{ - .contents = - { - { - "baz", - File::Regular{ - .executable = true, - .contents = "good day,\n\0\n\tworld!", - }, - }, - { - "quux", - File::Symlink{ - .target = "/over/there", - }, - }, - }, - }, - }, - }, - }; + auto files = memory_source_accessor::exampleComplex(); for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) { std::map cas; diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc new file mode 100644 index 000000000..542f32dc8 --- /dev/null +++ b/src/libutil-tests/memory-source-accessor.cc @@ -0,0 +1,116 @@ +#include + +#include "nix/util/bytes.hh" +#include "nix/util/memory-source-accessor.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +namespace memory_source_accessor { + +using File = MemorySourceAccessor::File; + +ref exampleSimple() +{ + auto sc = make_ref(); + sc->root = File{File::Regular{ + .executable = false, + .contents = to_owned(as_bytes("asdf")), + }}; + return sc; +} + +ref exampleComplex() +{ + auto files = make_ref(); + files->root = File::Directory{ + .contents{ + { + "foo", + File::Regular{ + .contents = to_owned(as_bytes("hello\n\0\n\tworld!")), + }, + }, + { + "bar", + File::Directory{ + .contents = + { + { + "baz", + File::Regular{ + .executable = true, + .contents = to_owned(as_bytes("good day,\n\0\n\tworld!")), + }, + }, + { + "quux", + File::Symlink{ + .target = "/over/there", + }, + }, + }, + }, + }, + }, + }; + return files; +} + +} // namespace memory_source_accessor + +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +class MemorySourceAccessorTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(MemorySourceAccessorJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + /* Cannot use `readJsonTest` because need to compare `root` field of + the source accessors for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + auto decoded = static_cast(encoded); + ASSERT_EQ(decoded.root, expected.root); + }); +} + +TEST_P(MemorySourceAccessorJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + MemorySourceAccessorJSON, + MemorySourceAccessorJsonTest, + ::testing::Values( + std::pair{ + "simple", + *memory_source_accessor::exampleSimple(), + }, + std::pair{ + "complex", + *memory_source_accessor::exampleComplex(), + })); + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index c75f4d90a..6c593eef8 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -63,6 +63,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'memory-source-accessor.cc', 'monitorfdhup.cc', 'nix_api_util.cc', 'nix_api_util_internal.cc', diff --git a/src/libutil/include/nix/util/bytes.hh b/src/libutil/include/nix/util/bytes.hh new file mode 100644 index 000000000..d1f037d7f --- /dev/null +++ b/src/libutil/include/nix/util/bytes.hh @@ -0,0 +1,41 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +static inline std::span as_bytes(std::string_view sv) noexcept +{ + return std::span{ + reinterpret_cast(sv.data()), + sv.size(), + }; +} + +static inline std::vector to_owned(std::span bytes) +{ + return std::vector{ + bytes.begin(), + bytes.end(), + }; +} + +/** + * @note this should be avoided, as arbitrary binary data in strings + * views, while allowed, is not really proper. Generally this should + * only be used as a stop-gap with other definitions that themselves + * should be converted to accept `std::span` or + * similar, directly. + */ +static inline std::string_view to_str(std::span sp) +{ + return std::string_view{ + reinterpret_cast(sp.data()), + sp.size(), + }; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index eba282fe1..9d7f3f80a 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -4,6 +4,7 @@ #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -25,7 +26,7 @@ struct MemorySourceAccessor : virtual SourceAccessor struct Regular { bool executable = false; - std::string contents; + std::vector contents; bool operator==(const Regular &) const = default; auto operator<=>(const Regular &) const = default; @@ -86,7 +87,13 @@ struct MemorySourceAccessor : virtual SourceAccessor */ File * open(const CanonPath & path, std::optional create); - SourcePath addFile(CanonPath path, std::string && contents); + SourcePath addFile(CanonPath path, std::vector && contents); + + /** + * Small wrapper of the other `addFile`, purely for convenience when + * the file in question to be added is a string. + */ + SourcePath addFile(CanonPath path, std::string_view contents); }; inline bool MemorySourceAccessor::File::Directory::operator==( @@ -121,4 +128,30 @@ struct MemorySink : FileSystemObjectSink void createSymlink(const CanonPath & path, const std::string & target) override; }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(MemorySourceAccessor::File::Regular) +JSON_IMPL(MemorySourceAccessor::File::Directory) +JSON_IMPL(MemorySourceAccessor::File::Symlink) +JSON_IMPL(MemorySourceAccessor::File) +JSON_IMPL(MemorySourceAccessor) diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 9a606e15d..89790f908 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -12,6 +12,7 @@ headers = files( 'array-from-string-literal.hh', 'base-n.hh', 'base-nix-32.hh', + 'bytes.hh', 'callback.hh', 'canon-path.hh', 'checked-arithmetic.hh', diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index a9ffb7746..78d54123c 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,4 +1,7 @@ #include "nix/util/memory-source-accessor.hh" +#include "nix/util/base-n.hh" +#include "nix/util/bytes.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -58,7 +61,7 @@ std::string MemorySourceAccessor::readFile(const CanonPath & path) if (!f) throw Error("file '%s' does not exist", path); if (auto * r = std::get_if(&f->raw)) - return r->contents; + return std::string{to_str(r->contents)}; else throw Error("file '%s' is not a regular file", path); } @@ -125,7 +128,7 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path) throw Error("file '%s' is not a symbolic link", path); } -SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::vector && contents) { // Create root directory automatically if necessary as a convenience. if (!root && !path.isRoot()) @@ -142,6 +145,11 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents return SourcePath{ref(shared_from_this()), path}; } +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string_view contents) +{ + return addFile(path, to_owned(as_bytes(contents))); +} + using File = MemorySourceAccessor::File; void MemorySink::createDirectory(const CanonPath & path) @@ -190,9 +198,10 @@ void CreateMemoryRegularFile::preallocateContents(uint64_t len) regularFile.contents.reserve(len); } -void CreateMemoryRegularFile::operator()(std::string_view data) +void CreateMemoryRegularFile::operator()(std::string_view data_) { - regularFile.contents += data; + auto data = as_bytes(data_); + regularFile.contents.insert(regularFile.contents.end(), data.begin(), data.end()); } void MemorySink::createSymlink(const CanonPath & path, const std::string & target) @@ -222,3 +231,106 @@ ref makeEmptySourceAccessor() } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +MemorySourceAccessor::File::Regular adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Regular{ + .executable = getBoolean(valueAt(obj, "executable")), + .contents = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "contents"))))), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Regular & val) +{ + json = { + {"executable", val.executable}, + {"contents", base64::encode(val.contents)}, + }; +} + +MemorySourceAccessor::File::Directory +adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Directory{ + .contents = valueAt(obj, "contents"), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Directory & val) +{ + json = { + {"contents", val.contents}, + }; +} + +MemorySourceAccessor::File::Symlink adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Symlink{ + .target = getString(valueAt(obj, "target")), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Symlink & val) +{ + json = { + {"target", val.target}, + }; +} + +MemorySourceAccessor::File adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + auto type = getString(valueAt(obj, "type")); + if (type == "regular") + return static_cast(json); + if (type == "directory") + return static_cast(json); + if (type == "symlink") + return static_cast(json); + else + throw Error("unknown type of file '%s'", type); +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor::File & val) +{ + std::visit( + overloaded{ + [&](const MemorySourceAccessor::File::Regular & r) { + json = r; + json["type"] = "regular"; + }, + [&](const MemorySourceAccessor::File::Directory & d) { + json = d; + json["type"] = "directory"; + }, + [&](const MemorySourceAccessor::File::Symlink & s) { + json = s; + json["type"] = "symlink"; + }, + }, + val.raw); +} + +MemorySourceAccessor adl_serializer::from_json(const json & json) +{ + MemorySourceAccessor res; + res.root = json; + return res; +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor & val) +{ + json = val.root; +} + +} // namespace nlohmann