From 2a194aa29e5e63c9320ede14fff07b135add9180 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 15:04:56 -0400 Subject: [PATCH 01/13] Make reading and writing derivations store methods This allows for different representations. --- src/libstore/derivations.cc | 27 ++++++++++++++++++--- src/libstore/include/nix/store/store-api.hh | 9 +++++-- src/libstore/store-api.cc | 2 +- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f44bf3e70..20f1d6ca1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -105,7 +105,7 @@ bool BasicDerivation::isBuiltin() const return builder.substr(0, 8) == "builtin:"; } -StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) +static auto infoForDerivation(Store & store, const Derivation & drv) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs.map) @@ -117,13 +117,32 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repa auto contents = drv.unparse(store, false); auto hash = hashString(HashAlgorithm::SHA256, contents); auto ca = TextInfo{.hash = hash, .references = references}; - auto path = store.makeFixedOutputPathFromCA(suffix, ca); + return std::tuple{ + suffix, + contents, + references, + store.makeFixedOutputPathFromCA(suffix, ca), + }; +} - if (readOnly || settings.readOnlyMode || (store.isValidPath(path) && !repair)) +StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) +{ + if (readOnly || settings.readOnlyMode) { + auto [_x, _y, _z, path] = infoForDerivation(store, drv); + return path; + } else + return store.writeDerivation(drv, repair); +} + +StorePath Store::writeDerivation(const Derivation & drv, RepairFlag repair) +{ + auto [suffix, contents, references, path] = infoForDerivation(*this, drv); + + if (isValidPath(path) && !repair) return path; StringSource s{contents}; - auto path2 = store.addToStoreFromDump( + auto path2 = addToStoreFromDump( s, suffix, FileSerialisationMethod::Flat, diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index d03e8e010..cc473adfd 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -778,15 +778,20 @@ public: */ Derivation derivationFromPath(const StorePath & drvPath); + /** + * Write a derivation to the Nix store, and return its path. + */ + virtual StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair); + /** * Read a derivation (which must already be valid). */ - Derivation readDerivation(const StorePath & drvPath); + virtual Derivation readDerivation(const StorePath & drvPath); /** * Read a derivation from a potentially invalid path. */ - Derivation readInvalidDerivation(const StorePath & drvPath); + virtual Derivation readInvalidDerivation(const StorePath & drvPath); /** * @param [out] out Place in here the set of all store paths in the diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cdca6a763..88f79688d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1163,7 +1163,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) // resolved derivation, so we need to get it first auto resolvedDrv = drv.tryResolve(*this); if (resolvedDrv) - return writeDerivation(*this, *resolvedDrv, NoRepair, true); + return ::nix::writeDerivation(*this, *resolvedDrv, NoRepair, true); } return path; From 9afb7e508277f185c1c7ada50e886e74672b9947 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Sep 2025 00:29:21 -0400 Subject: [PATCH 02/13] `nlohmann::json` instance and JSON Schema for `MemorySourceAccessor` As documented, this for file system objects themselves, since `MemorySourceAccessor` is an implementation detail. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/file-system-object.md | 21 +++ .../json/fixup-json-schema-generated-doc.sed | 1 + doc/manual/source/protocols/json/meson.build | 1 + .../json/schema/file-system-object-v1 | 1 + .../json/schema/file-system-object-v1.yaml | 65 ++++++++++ src/json-schema-checks/file-system-object | 1 + src/json-schema-checks/meson.build | 10 +- src/json-schema-checks/package.nix | 1 + .../data/memory-source-accessor/complex.json | 24 ++++ .../data/memory-source-accessor/simple.json | 5 + src/libutil-tests/git.cc | 41 +----- src/libutil-tests/memory-source-accessor.cc | 116 +++++++++++++++++ src/libutil-tests/meson.build | 1 + src/libutil/include/nix/util/bytes.hh | 41 ++++++ .../nix/util/memory-source-accessor.hh | 37 +++++- src/libutil/include/nix/util/meson.build | 1 + src/libutil/memory-source-accessor.cc | 120 +++++++++++++++++- 19 files changed, 447 insertions(+), 42 deletions(-) create mode 100644 doc/manual/source/protocols/json/file-system-object.md create mode 120000 doc/manual/source/protocols/json/schema/file-system-object-v1 create mode 100644 doc/manual/source/protocols/json/schema/file-system-object-v1.yaml create mode 120000 src/json-schema-checks/file-system-object create mode 100644 src/libutil-tests/data/memory-source-accessor/complex.json create mode 100644 src/libutil-tests/data/memory-source-accessor/simple.json create mode 100644 src/libutil-tests/memory-source-accessor.cc create mode 100644 src/libutil/include/nix/util/bytes.hh diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 30486869e..fc64cee86 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -34,6 +34,7 @@ mkMesonDerivation (finalAttrs: { (fileset.unions [ ../../.version # For example JSON + ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash # Too many different types of files to filter for now ../../doc/manual diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index f74ed7043..6cb816af6 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -117,6 +117,7 @@ - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) + - [File System Object](protocols/json/file-system-object.md) - [Hash](protocols/json/hash.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) diff --git a/doc/manual/source/protocols/json/file-system-object.md b/doc/manual/source/protocols/json/file-system-object.md new file mode 100644 index 000000000..6517d73ca --- /dev/null +++ b/doc/manual/source/protocols/json/file-system-object.md @@ -0,0 +1,21 @@ +{{#include file-system-object-v1-fixed.md}} + +## Examples + +### Simple + +```json +{{#include schema/file-system-object-v1/simple.json}} +``` + +### Complex + +```json +{{#include schema/file-system-object-v1/complex.json}} +``` + + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 126e666e9..397c2dc5d 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -11,4 +11,5 @@ s/\\`/`/g # # As we have more such relative links, more replacements of this nature # should appear below. +s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 44795599c..d5c560fd3 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -9,6 +9,7 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false) json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ + 'file-system-object-v1', 'hash-v1', 'derivation-v3', ] diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1 b/doc/manual/source/protocols/json/schema/file-system-object-v1 new file mode 120000 index 000000000..cbb21a10d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1 @@ -0,0 +1 @@ +../../../../../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml new file mode 100644 index 000000000..c7154b18d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml @@ -0,0 +1,65 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json +title: File System Object +description: | + This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md). + + The schema is recursive because file system objects contain other file system objects. +type: object +required: ["type"] +properties: + type: + type: string + enum: ["regular", "symlink", "directory"] + +# Enforce conditional structure based on `type` +anyOf: + - $ref: "#/$defs/regular" + required: ["type", "contents"] + + - $ref: "#/$defs/symlink" + required: ["type", "target"] + + - $ref: "#/$defs/directory" + required: ["type", "contents"] + +"$defs": + regular: + title: Regular File + required: ["contents"] + properties: + type: + const: "regular" + contents: + type: string + description: Base64-encoded file contents + executable: + type: boolean + description: Whether the file is executable. + default: false + additionalProperties: false + + symlink: + title: Symbolic Link + required: ["target"] + properties: + type: + const: "symlink" + target: + type: string + description: Target path of the symlink. + additionalProperties: false + + directory: + title: Directory + required: ["contents"] + properties: + type: + const: "directory" + contents: + type: object + description: | + Map of names to nested file system objects (for type=directory) + additionalProperties: + $ref: "#" + additionalProperties: false diff --git a/src/json-schema-checks/file-system-object b/src/json-schema-checks/file-system-object new file mode 120000 index 000000000..b26e030c9 --- /dev/null +++ b/src/json-schema-checks/file-system-object @@ -0,0 +1 @@ +../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index ebd6f6b2b..1ab7f0dc3 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema' # Get all example files schemas = [ + { + 'stem' : 'file-system-object', + 'schema' : schema_dir / 'file-system-object-v1.yaml', + 'files' : [ + 'simple.json', + 'complex.json', + ], + }, { 'stem' : 'hash', 'schema' : schema_dir / 'hash-v1.yaml', @@ -64,8 +72,6 @@ foreach schema : schemas stem + '-schema-valid', jv, args : [ - '--map', - './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml', 'http://json-schema.org/draft-04/schema', schema_file, ], diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 41458adb8..6eddbd5dd 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: { fileset = lib.fileset.unions [ ../../.version ../../doc/manual/source/protocols/json/schema + ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/derivation ./. diff --git a/src/libutil-tests/data/memory-source-accessor/complex.json b/src/libutil-tests/data/memory-source-accessor/complex.json new file mode 100644 index 000000000..082956768 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/complex.json @@ -0,0 +1,24 @@ +{ + "contents": { + "bar": { + "contents": { + "baz": { + "contents": "Z29vZCBkYXksCg==", + "executable": true, + "type": "regular" + }, + "quux": { + "target": "/over/there", + "type": "symlink" + } + }, + "type": "directory" + }, + "foo": { + "contents": "aGVsbG8K", + "executable": false, + "type": "regular" + } + }, + "type": "directory" +} diff --git a/src/libutil-tests/data/memory-source-accessor/simple.json b/src/libutil-tests/data/memory-source-accessor/simple.json new file mode 100644 index 000000000..8c61fa730 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/simple.json @@ -0,0 +1,5 @@ +{ + "contents": "YXNkZg==", + "executable": false, + "type": "regular" +} diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index 6180a4cfc..f761c4433 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -224,42 +224,15 @@ TEST_F(GitTest, tree_sha256_write) }); } +namespace memory_source_accessor { + +extern ref exampleComplex(); + +} + TEST_F(GitTest, both_roundrip) { - using File = MemorySourceAccessor::File; - - auto files = make_ref(); - files->root = File::Directory{ - .contents{ - { - "foo", - File::Regular{ - .contents = "hello\n\0\n\tworld!", - }, - }, - { - "bar", - File::Directory{ - .contents = - { - { - "baz", - File::Regular{ - .executable = true, - .contents = "good day,\n\0\n\tworld!", - }, - }, - { - "quux", - File::Symlink{ - .target = "/over/there", - }, - }, - }, - }, - }, - }, - }; + auto files = memory_source_accessor::exampleComplex(); for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) { std::map cas; diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc new file mode 100644 index 000000000..542f32dc8 --- /dev/null +++ b/src/libutil-tests/memory-source-accessor.cc @@ -0,0 +1,116 @@ +#include + +#include "nix/util/bytes.hh" +#include "nix/util/memory-source-accessor.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +namespace memory_source_accessor { + +using File = MemorySourceAccessor::File; + +ref exampleSimple() +{ + auto sc = make_ref(); + sc->root = File{File::Regular{ + .executable = false, + .contents = to_owned(as_bytes("asdf")), + }}; + return sc; +} + +ref exampleComplex() +{ + auto files = make_ref(); + files->root = File::Directory{ + .contents{ + { + "foo", + File::Regular{ + .contents = to_owned(as_bytes("hello\n\0\n\tworld!")), + }, + }, + { + "bar", + File::Directory{ + .contents = + { + { + "baz", + File::Regular{ + .executable = true, + .contents = to_owned(as_bytes("good day,\n\0\n\tworld!")), + }, + }, + { + "quux", + File::Symlink{ + .target = "/over/there", + }, + }, + }, + }, + }, + }, + }; + return files; +} + +} // namespace memory_source_accessor + +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +class MemorySourceAccessorTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(MemorySourceAccessorJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + /* Cannot use `readJsonTest` because need to compare `root` field of + the source accessors for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + auto decoded = static_cast(encoded); + ASSERT_EQ(decoded.root, expected.root); + }); +} + +TEST_P(MemorySourceAccessorJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + MemorySourceAccessorJSON, + MemorySourceAccessorJsonTest, + ::testing::Values( + std::pair{ + "simple", + *memory_source_accessor::exampleSimple(), + }, + std::pair{ + "complex", + *memory_source_accessor::exampleComplex(), + })); + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index c75f4d90a..6c593eef8 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -63,6 +63,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'memory-source-accessor.cc', 'monitorfdhup.cc', 'nix_api_util.cc', 'nix_api_util_internal.cc', diff --git a/src/libutil/include/nix/util/bytes.hh b/src/libutil/include/nix/util/bytes.hh new file mode 100644 index 000000000..d1f037d7f --- /dev/null +++ b/src/libutil/include/nix/util/bytes.hh @@ -0,0 +1,41 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +static inline std::span as_bytes(std::string_view sv) noexcept +{ + return std::span{ + reinterpret_cast(sv.data()), + sv.size(), + }; +} + +static inline std::vector to_owned(std::span bytes) +{ + return std::vector{ + bytes.begin(), + bytes.end(), + }; +} + +/** + * @note this should be avoided, as arbitrary binary data in strings + * views, while allowed, is not really proper. Generally this should + * only be used as a stop-gap with other definitions that themselves + * should be converted to accept `std::span` or + * similar, directly. + */ +static inline std::string_view to_str(std::span sp) +{ + return std::string_view{ + reinterpret_cast(sp.data()), + sp.size(), + }; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index eba282fe1..9d7f3f80a 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -4,6 +4,7 @@ #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -25,7 +26,7 @@ struct MemorySourceAccessor : virtual SourceAccessor struct Regular { bool executable = false; - std::string contents; + std::vector contents; bool operator==(const Regular &) const = default; auto operator<=>(const Regular &) const = default; @@ -86,7 +87,13 @@ struct MemorySourceAccessor : virtual SourceAccessor */ File * open(const CanonPath & path, std::optional create); - SourcePath addFile(CanonPath path, std::string && contents); + SourcePath addFile(CanonPath path, std::vector && contents); + + /** + * Small wrapper of the other `addFile`, purely for convenience when + * the file in question to be added is a string. + */ + SourcePath addFile(CanonPath path, std::string_view contents); }; inline bool MemorySourceAccessor::File::Directory::operator==( @@ -121,4 +128,30 @@ struct MemorySink : FileSystemObjectSink void createSymlink(const CanonPath & path, const std::string & target) override; }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(MemorySourceAccessor::File::Regular) +JSON_IMPL(MemorySourceAccessor::File::Directory) +JSON_IMPL(MemorySourceAccessor::File::Symlink) +JSON_IMPL(MemorySourceAccessor::File) +JSON_IMPL(MemorySourceAccessor) diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 9a606e15d..89790f908 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -12,6 +12,7 @@ headers = files( 'array-from-string-literal.hh', 'base-n.hh', 'base-nix-32.hh', + 'bytes.hh', 'callback.hh', 'canon-path.hh', 'checked-arithmetic.hh', diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index a9ffb7746..78d54123c 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,4 +1,7 @@ #include "nix/util/memory-source-accessor.hh" +#include "nix/util/base-n.hh" +#include "nix/util/bytes.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -58,7 +61,7 @@ std::string MemorySourceAccessor::readFile(const CanonPath & path) if (!f) throw Error("file '%s' does not exist", path); if (auto * r = std::get_if(&f->raw)) - return r->contents; + return std::string{to_str(r->contents)}; else throw Error("file '%s' is not a regular file", path); } @@ -125,7 +128,7 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path) throw Error("file '%s' is not a symbolic link", path); } -SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::vector && contents) { // Create root directory automatically if necessary as a convenience. if (!root && !path.isRoot()) @@ -142,6 +145,11 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents return SourcePath{ref(shared_from_this()), path}; } +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string_view contents) +{ + return addFile(path, to_owned(as_bytes(contents))); +} + using File = MemorySourceAccessor::File; void MemorySink::createDirectory(const CanonPath & path) @@ -190,9 +198,10 @@ void CreateMemoryRegularFile::preallocateContents(uint64_t len) regularFile.contents.reserve(len); } -void CreateMemoryRegularFile::operator()(std::string_view data) +void CreateMemoryRegularFile::operator()(std::string_view data_) { - regularFile.contents += data; + auto data = as_bytes(data_); + regularFile.contents.insert(regularFile.contents.end(), data.begin(), data.end()); } void MemorySink::createSymlink(const CanonPath & path, const std::string & target) @@ -222,3 +231,106 @@ ref makeEmptySourceAccessor() } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +MemorySourceAccessor::File::Regular adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Regular{ + .executable = getBoolean(valueAt(obj, "executable")), + .contents = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "contents"))))), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Regular & val) +{ + json = { + {"executable", val.executable}, + {"contents", base64::encode(val.contents)}, + }; +} + +MemorySourceAccessor::File::Directory +adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Directory{ + .contents = valueAt(obj, "contents"), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Directory & val) +{ + json = { + {"contents", val.contents}, + }; +} + +MemorySourceAccessor::File::Symlink adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Symlink{ + .target = getString(valueAt(obj, "target")), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Symlink & val) +{ + json = { + {"target", val.target}, + }; +} + +MemorySourceAccessor::File adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + auto type = getString(valueAt(obj, "type")); + if (type == "regular") + return static_cast(json); + if (type == "directory") + return static_cast(json); + if (type == "symlink") + return static_cast(json); + else + throw Error("unknown type of file '%s'", type); +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor::File & val) +{ + std::visit( + overloaded{ + [&](const MemorySourceAccessor::File::Regular & r) { + json = r; + json["type"] = "regular"; + }, + [&](const MemorySourceAccessor::File::Directory & d) { + json = d; + json["type"] = "directory"; + }, + [&](const MemorySourceAccessor::File::Symlink & s) { + json = s; + json["type"] = "symlink"; + }, + }, + val.raw); +} + +MemorySourceAccessor adl_serializer::from_json(const json & json) +{ + MemorySourceAccessor res; + res.root = json; + return res; +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor & val) +{ + json = val.root; +} + +} // namespace nlohmann From 2d4c7d3d038f4373da21de918b33b118dafab693 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 16:00:27 -0400 Subject: [PATCH 03/13] Make Dummy store store derivations separately This makes for more efficiency. Once we have JSON for the dummy store, it will also make for better JSON, too. --- src/libstore-tests/write-derivation.cc | 4 +- src/libstore/dummy-store.cc | 103 ++++++++++++++---- .../include/nix/store/dummy-store-impl.hh | 9 +- 3 files changed, 93 insertions(+), 23 deletions(-) diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index 3f7de05d3..c320f92fa 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -50,8 +50,8 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) EXPECT_EQ(path1, path2); EXPECT_THAT( [&] { writeDerivation(*store, drv, Repair); }, - ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher( - "operation 'addToStoreFromDump' is not supported by store 'dummy://'"))); + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("operation 'writeDerivation' is not supported by store 'dummy://'"))); } } // namespace nix diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 509b7a0b1..4608d3077 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -137,12 +137,35 @@ struct DummyStoreImpl : DummyStore void queryPathInfoUncached( const StorePath & path, Callback> callback) noexcept override { - bool visited = contents.cvisit(path, [&](const auto & kv) { - callback(std::make_shared(StorePath{kv.first}, kv.second.info)); - }); + if (path.isDerivation()) { + if (derivations.cvisit(path, [&](const auto & kv) { + /* compute path info on demand */ + auto accessor = make_ref(); + accessor->root = MemorySourceAccessor::File::Regular{ + .contents = kv.second.unparse(*this, false), + }; + auto narHash = hashPath( + {accessor, CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + auto info = + std::make_shared(StorePath{kv.first}, UnkeyedValidPathInfo{narHash.hash}); + info->narSize = narHash.numBytesDigested; + info->ca = ContentAddress{ + .method = ContentAddressMethod::Raw::Text, + .hash = hashString( + HashAlgorithm::SHA256, + std::get(accessor->root->raw).contents), + }; + callback(std::move(info)); + })) + return; + } else { + if (contents.cvisit(path, [&](const auto & kv) { + callback(std::make_shared(StorePath{kv.first}, kv.second.info)); + })) + return; + } - if (!visited) - callback(nullptr); + callback(nullptr); } /** @@ -169,18 +192,25 @@ struct DummyStoreImpl : DummyStore if (checkSigs) throw Error("checking signatures is not supported for '%s' store", config->getHumanReadableURI()); - auto temp = make_ref(); - MemorySink tempSink{*temp}; + auto accessor = make_ref(); + MemorySink tempSink{*accessor}; parseDump(tempSink, source); auto path = info.path; - auto accessor = make_ref(std::move(*temp)); - contents.insert( - {path, - PathInfoAndContents{ - std::move(info), - accessor, - }}); + if (info.path.isDerivation()) { + warn("back compat supporting `addToStore` for inserting derivations in dummy store"); + writeDerivation( + parseDerivation(*this, accessor->readFile(CanonPath::root), Derivation::nameFromPath(info.path))); + return; + } + + contents.insert({ + path, + PathInfoAndContents{ + std::move(info), + accessor, + }, + }); wholeStoreView->addObject(path.to_string(), accessor); } @@ -193,6 +223,9 @@ struct DummyStoreImpl : DummyStore const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override { + if (isDerivation(name)) + throw Error("Do not insert derivation into dummy store with `addToStoreFromDump`"); + if (config->readOnly) unsupported("addToStoreFromDump"); @@ -239,17 +272,47 @@ struct DummyStoreImpl : DummyStore auto path = info.path; auto accessor = make_ref(std::move(*temp)); - contents.insert( - {path, - PathInfoAndContents{ - std::move(info), - accessor, - }}); + contents.insert({ + path, + PathInfoAndContents{ + std::move(info), + accessor, + }, + }); wholeStoreView->addObject(path.to_string(), accessor); return path; } + StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair) override + { + auto drvPath = ::nix::writeDerivation(*this, drv, repair, /*readonly=*/true); + + if (!derivations.contains(drvPath) || repair) { + if (config->readOnly) + unsupported("writeDerivation"); + derivations.insert({drvPath, drv}); + } + + return drvPath; + } + + Derivation readDerivation(const StorePath & drvPath) override + { + if (std::optional res = getConcurrent(derivations, drvPath)) + return *res; + else + throw Error("derivation '%s' is not valid", printStorePath(drvPath)); + } + + /** + * No such thing as an "invalid derivation" with the dummy store + */ + Derivation readInvalidDerivation(const StorePath & drvPath) override + { + return readDerivation(drvPath); + } + void registerDrvOutput(const Realisation & output) override { auto ref = make_ref(output); diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 4c9f54e98..137f81c9b 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/dummy-store.hh" +#include "nix/store/derivations.hh" #include @@ -25,11 +26,17 @@ struct DummyStore : virtual Store }; /** - * This is map conceptually owns the file system objects for each + * This map conceptually owns the file system objects for each * store object. */ boost::concurrent_flat_map contents; + /** + * This map conceptually owns every derivation, allowing us to + * avoid "on-disk drv format" serialization round-trips. + */ + boost::concurrent_flat_map derivations; + /** * The build trace maps the pair of a content-addressing (fixed or * floating) derivations an one of its output to a From 34de68d2604f12ab625b1207fef949db2bc284da Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Oct 2025 11:37:14 -0400 Subject: [PATCH 04/13] Optimize `DummyStore::isValidPathUncached` See the API docs for the rationale of why this is needed. --- src/libstore/dummy-store.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 4608d3077..5924f897c 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -168,6 +168,15 @@ struct DummyStoreImpl : DummyStore callback(nullptr); } + /** + * Do this to avoid `queryPathInfoUncached` computing `PathInfo` + * that we don't need just to return a `bool`. + */ + bool isValidPathUncached(const StorePath & path) override + { + return path.isDerivation() ? derivations.contains(path) : Store::isValidPathUncached(path); + } + /** * The dummy store is incapable of *not* trusting! :) */ From 0bec9d07167a450dda763741cfcef0c6ca9cc7fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 00:15:24 -0400 Subject: [PATCH 05/13] `nlohmann::json` instance and JSON Schema for `ContentAddress` Co-authored-by: Robert Hensing --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/json/content-address.md | 21 ++++ .../json/fixup-json-schema-generated-doc.sed | 3 + doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/content-address-v1 | 1 + .../json/schema/content-address-v1.yaml | 51 ++++++++++ .../protocols/json/schema/derivation-v3.yaml | 13 +-- src/json-schema-checks/content-address | 1 + src/json-schema-checks/meson.build | 10 +- src/json-schema-checks/package.nix | 1 + src/libstore-tests/content-address.cc | 97 +++++++++++++++---- .../data/content-address/nar.json | 8 ++ .../data/content-address/text.json | 8 ++ src/libstore/content-address.cc | 34 +++++++ .../include/nix/store/content-address.hh | 12 +++ 16 files changed, 232 insertions(+), 31 deletions(-) create mode 100644 doc/manual/source/protocols/json/content-address.md create mode 120000 doc/manual/source/protocols/json/schema/content-address-v1 create mode 100644 doc/manual/source/protocols/json/schema/content-address-v1.yaml create mode 120000 src/json-schema-checks/content-address create mode 100644 src/libstore-tests/data/content-address/nar.json create mode 100644 src/libstore-tests/data/content-address/text.json diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 30486869e..3ce21adfb 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -35,6 +35,7 @@ mkMesonDerivation (finalAttrs: { ../../.version # For example JSON ../../src/libutil-tests/data/hash + ../../src/libstore-tests/data/content-address # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index f74ed7043..e1e7f0692 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -118,6 +118,7 @@ - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) - [Hash](protocols/json/hash.md) + - [Pseudo Content Address](protocols/json/content-address.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) diff --git a/doc/manual/source/protocols/json/content-address.md b/doc/manual/source/protocols/json/content-address.md new file mode 100644 index 000000000..7a0a440da --- /dev/null +++ b/doc/manual/source/protocols/json/content-address.md @@ -0,0 +1,21 @@ +{{#include content-address-v1-fixed.md}} + +## Examples + +### [Text](file:///home/jcericson/src/nix/4/build-linux-clang/src/nix-manual/manual/store/store-object/content-address.html#method-text) method + +```json +{{#include schema/content-address-v1/text.json}} +``` + +### [Nix Archive](file:///home/jcericson/src/nix/4/build-linux-clang/src/nix-manual/manual/store/store-object/content-address.html#method-nix-archive) method + +```json +{{#include schema/content-address-v1/nar.json}} +``` + + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 126e666e9..27895d42a 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -12,3 +12,6 @@ s/\\`/`/g # As we have more such relative links, more replacements of this nature # should appear below. s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g +s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g +s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g +s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 44795599c..d45f1c010 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -10,6 +10,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ 'hash-v1', + 'content-address-v1', 'derivation-v3', ] diff --git a/doc/manual/source/protocols/json/schema/content-address-v1 b/doc/manual/source/protocols/json/schema/content-address-v1 new file mode 120000 index 000000000..35a0dd865 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/content-address-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/content-address \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/content-address-v1.yaml b/doc/manual/source/protocols/json/schema/content-address-v1.yaml new file mode 100644 index 000000000..56da79225 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/content-address-v1.yaml @@ -0,0 +1,51 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/content-address-v1.json +title: Pesudo Content Address +description: | + This schema describes the JSON representation of Nix's `ContentAddress` type. + + This data type is not a simple as a looks, and therefore called a *pseudo* content address. + See the description of the `hash` field for why this is. + + When creating the store path of a content-addressed store object, the `hash` from this will be combined with the references of the store object in order to create a content-address that is properly sensitive to the entirety of the store object — file system objects and references alike. + So it is that store path that is arguably the *true* content-address for content-addressed store paths, not this data type. + + The only problem with that is that store paths are truncated in various ways from the underlying hashes, so their cryptographic strength can be reasonably doubted. + Given that uncertain cryptographic strength, we do need something else. + Currently this in conjunction with the reference list captures all the content securely, more concisely. + (Though the reference list is still arbitrarily-sized, it is presumably in practice still way smaller than the file system object data). + + > **Note** + > + > A hypothetical hash of both of those which is *not* so truncated would be even better, as it would be secure and fixed-size. + > A hypothetical new method content-addressing store objects where we compute the store paths just from such a hash, so we don't need to "dereference" the hash to get the underlying reference list and rehash it, would be better still. +type: object +properties: + method: + "$ref": "#/$defs/method" + hash: + title: Content Address + description: | + This would be the content-address itself. + + For all current methods, this is just a content address of the file system object of the store object, [as described in the store chapter](@docroot@/store/store-object/content-address.md), and not of the store object as a whole. + In particular, the references of the store object are *not* taken into account with this hash (and currently-supported methods). + "$ref": "./hash-v1.yaml" +required: +- method +- hash +additionalProperties: false +"$defs": + method: + type: string + enum: [flat, nar, text, git] + title: Content-Addressing Method + description: | + A string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) (provided the contents are a single file) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index 7c92d475d..c5720ec6d 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -154,19 +154,10 @@ properties: The output path, if known in advance. method: - type: string - title: Content addressing method - enum: [flat, nar, text, git] + "$ref": "./content-address-v1.yaml#/$defs/method" description: | For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. - - Valid method strings are: - - - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) - - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) - - [`text`](@docroot@/store/store-object/content-address.md#method-text) - - [`git`](@docroot@/store/store-object/content-address.md#method-git) - + See the linked original definition for further details. hashAlgo: title: Hash algorithm "$ref": "./hash-v1.yaml#/$defs/algorithm" diff --git a/src/json-schema-checks/content-address b/src/json-schema-checks/content-address new file mode 120000 index 000000000..194a265a1 --- /dev/null +++ b/src/json-schema-checks/content-address @@ -0,0 +1 @@ +../../src/libstore-tests/data/content-address \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index ebd6f6b2b..5ac5a6e1d 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -30,6 +30,14 @@ schemas = [ 'blake3-base64.json', ], }, + { + 'stem' : 'content-address', + 'schema' : schema_dir / 'content-address-v1.yaml', + 'files' : [ + 'text.json', + 'nar.json', + ], + }, { 'stem' : 'derivation', 'schema' : schema_dir / 'derivation-v3.yaml', @@ -64,8 +72,6 @@ foreach schema : schemas stem + '-schema-valid', jv, args : [ - '--map', - './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml', 'http://json-schema.org/draft-04/schema', schema_file, ], diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 41458adb8..9986160ac 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -21,6 +21,7 @@ mkMesonDerivation (finalAttrs: { ../../.version ../../doc/manual/source/protocols/json/schema ../../src/libutil-tests/data/hash + ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/derivation ./. ]; diff --git a/src/libstore-tests/content-address.cc b/src/libstore-tests/content-address.cc index 51d591c38..0474fb2e0 100644 --- a/src/libstore-tests/content-address.cc +++ b/src/libstore-tests/content-address.cc @@ -1,6 +1,7 @@ #include #include "nix/store/content-address.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { @@ -8,33 +9,93 @@ namespace nix { * ContentAddressMethod::parse, ContentAddressMethod::render * --------------------------------------------------------------------------*/ -TEST(ContentAddressMethod, testRoundTripPrintParse_1) +static auto methods = ::testing::Values( + std::pair{ContentAddressMethod::Raw::Text, "text"}, + std::pair{ContentAddressMethod::Raw::Flat, "flat"}, + std::pair{ContentAddressMethod::Raw::NixArchive, "nar"}, + std::pair{ContentAddressMethod::Raw::Git, "git"}); + +struct ContentAddressMethodTest : ::testing::Test, + ::testing::WithParamInterface> +{}; + +TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1) { - for (ContentAddressMethod cam : { - ContentAddressMethod::Raw::Text, - ContentAddressMethod::Raw::Flat, - ContentAddressMethod::Raw::NixArchive, - ContentAddressMethod::Raw::Git, - }) { - EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); - } + auto & [cam, _] = GetParam(); + EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); } -TEST(ContentAddressMethod, testRoundTripPrintParse_2) +TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2) { - for (const std::string_view camS : { - "text", - "flat", - "nar", - "git", - }) { - EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); - } + auto & [cam, camS] = GetParam(); + EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); } +INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods); + TEST(ContentAddressMethod, testParseContentAddressMethodOptException) { EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError); } +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +class ContentAddressTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "content-address"; + +public: + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct ContentAddressJsonTest : ContentAddressTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(ContentAddressJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(ContentAddressJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + ContentAddressJSON, + ContentAddressJsonTest, + ::testing::Values( + std::pair{ + "text", + ContentAddress{ + .method = ContentAddressMethod::Raw::Text, + .hash = hashString(HashAlgorithm::SHA256, "asdf"), + }, + }, + std::pair{ + "nar", + ContentAddress{ + .method = ContentAddressMethod::Raw::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "qwer"), + }, + })); + } // namespace nix diff --git a/src/libstore-tests/data/content-address/nar.json b/src/libstore-tests/data/content-address/nar.json new file mode 100644 index 000000000..21e065cd3 --- /dev/null +++ b/src/libstore-tests/data/content-address/nar.json @@ -0,0 +1,8 @@ +{ + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "9vLqj0XYoFfJVmoz+ZR02i5camYE1zYSFlDicwxvsKM=" + }, + "method": "nar" +} diff --git a/src/libstore-tests/data/content-address/text.json b/src/libstore-tests/data/content-address/text.json new file mode 100644 index 000000000..04bc8ac20 --- /dev/null +++ b/src/libstore-tests/data/content-address/text.json @@ -0,0 +1,8 @@ +{ + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts=" + }, + "method": "text" +} diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 9a57e3aa6..497c2c5b4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,6 +1,7 @@ #include "nix/util/args.hh" #include "nix/store/content-address.hh" #include "nix/util/split.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -300,3 +301,36 @@ Hash ContentAddressWithReferences::getHash() const } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +ContentAddressMethod adl_serializer::from_json(const json & json) +{ + return ContentAddressMethod::parse(getString(json)); +} + +void adl_serializer::to_json(json & json, const ContentAddressMethod & m) +{ + json = m.render(); +} + +ContentAddress adl_serializer::from_json(const json & json) +{ + auto obj = getObject(json); + return { + .method = adl_serializer::from_json(valueAt(obj, "method")), + .hash = valueAt(obj, "hash"), + }; +} + +void adl_serializer::to_json(json & json, const ContentAddress & ca) +{ + json = { + {"method", ca.method}, + {"hash", ca.hash}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/content-address.hh b/src/libstore/include/nix/store/content-address.hh index 0a3dc79bd..41ccc69ae 100644 --- a/src/libstore/include/nix/store/content-address.hh +++ b/src/libstore/include/nix/store/content-address.hh @@ -6,6 +6,7 @@ #include "nix/store/path.hh" #include "nix/util/file-content-address.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -308,4 +309,15 @@ struct ContentAddressWithReferences Hash getHash() const; }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::ContentAddressMethod) +JSON_IMPL(nix::ContentAddress) From f53c8b8c903dce8b9763a05884f0830a8c732fdf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Oct 2025 13:03:18 -0400 Subject: [PATCH 06/13] Split realisation protocol unit tests This will allow us to more accurately test dropping support for dependent realisations. --- src/libstore-tests/common-protocol.cc | 19 +++++++++++++-- .../common-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/common-protocol/realisation.bin | Bin 520 -> 384 bytes .../serve-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/serve-protocol/realisation.bin | Bin 520 -> 384 bytes .../worker-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/worker-protocol/realisation.bin | Bin 520 -> 384 bytes src/libstore-tests/serve-protocol.cc | 17 +++++++++++++ src/libstore-tests/worker-protocol.cc | 23 +++++++++++++++--- 9 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 src/libstore-tests/data/common-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-with-deps.bin diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 2c001957b..7c40e8cdb 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -114,13 +114,28 @@ CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +CHARACTERIZATION_TEST( + realisation_with_deps, + "realisation-with-deps", + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/data/common-protocol/realisation-with-deps.bin b/src/libstore-tests/data/common-protocol/realisation-with-deps.bin new file mode 100644 index 0000000000000000000000000000000000000000..54a78b64ebcf0726bd7da506434a316c7336e6e9 GIT binary patch literal 320 zcmXqFWB`L|rIgfy)V!3`ypo{Q#GK6H#FEVXykaG*YNg_gL?cr(E3-5UGs`r~)I=i- zBjco$L_;&vR0A^ubF;J*gOpSgV>7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -104,6 +113,14 @@ VERSIONED_CHARACTERIZATION_TEST( .outputName = "baz", }, }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index c4afde3bd..8f70e937b 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -150,13 +150,30 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -172,7 +189,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, }, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, From 15f3abb35e90db5131ddb57be72cf248fa5b07d9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 18:35:19 -0400 Subject: [PATCH 07/13] Remove dependent realisations This progress on #11896. It introduces some issues temporarily which will be fixed when #11928 is fixed. --- .../include/nix/store/tests/protocol.hh | 22 +++-- src/libstore-tests/common-protocol.cc | 34 ++++---- src/libstore-tests/realisation.cc | 81 ++++++++--------- src/libstore-tests/serve-protocol.cc | 12 +-- src/libstore-tests/worker-protocol.cc | 12 +-- src/libstore/build/derivation-goal.cc | 5 -- .../build/drv-output-substitution-goal.cc | 24 ------ src/libstore/ca-specific-schema.sql | 27 ------ src/libstore/include/nix/store/realisation.hh | 16 ---- src/libstore/include/nix/store/store-api.hh | 3 - src/libstore/local-store.cc | 49 ----------- src/libstore/misc.cc | 59 ------------- src/libstore/realisation.cc | 86 +------------------ src/libstore/restricted-store.cc | 2 +- src/libstore/store-api.cc | 40 ++++----- .../ca/duplicate-realisation-in-closure.sh | 5 ++ tests/functional/ca/substitute.sh | 5 +- 17 files changed, 98 insertions(+), 384 deletions(-) diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 5b57c6585..a1acf2fee 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -69,14 +69,20 @@ public: } }; -#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ - TEST_F(FIXTURE, NAME##_read) \ - { \ - readProtoTest(STEM, VERSION, VALUE); \ - } \ - TEST_F(FIXTURE, NAME##_write) \ - { \ - writeProtoTest(STEM, VERSION, VALUE); \ +#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_read) \ + { \ + readProtoTest(STEM, VERSION, VALUE); \ } +#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_write) \ + { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) + } // namespace nix diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 7c40e8cdb..16030c8f1 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -46,16 +46,22 @@ public: } }; -#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ - TEST_F(CommonProtoTest, NAME##_read) \ - { \ - readProtoTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_write) \ - { \ - writeProtoTest(STEM, VALUE); \ +#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_read) \ + { \ + readProtoTest(STEM, VALUE); \ } +#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_write) \ + { \ + writeProtoTest(STEM, VALUE); \ + } + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) + CHARACTERIZATION_TEST( string, "string", @@ -132,7 +138,7 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +READ_CHARACTERIZATION_TEST( realisation_with_deps, "realisation-with-deps", (std::tuple{ @@ -140,16 +146,6 @@ CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d16049bc5..66f4707e9 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -44,54 +44,45 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } +Realisation simple{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + }, + { + .drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), + .outputName = "foo", + }, +}; + INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, - ([] { - Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - }; - return ::testing::Values( - std::pair{ - "simple", - simple, - }, - std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = {"asdfasdfasdf"}; - return r; - }()}, - std::pair{ - "with-dependent-realisations", - [&] { - auto r = simple; - r.dependentRealisations = {{ - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }}; - return r; - }(), - }); - } + ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = {"asdfasdfasdf"}; + return r; + }(), + })); - ())); +/** + * We no longer have a notion of "dependent realisations", but we still + * want to parse old realisation files. So make this just be a read test + * (no write direction), accordingly. + */ +TEST_F(RealisationTest, dependent_realisations_from_json) +{ + readJsonTest("with-dependent-realisations", simple); +} } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index a7b69821c..0821bb99b 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -115,7 +115,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, realisation_with_deps, "realisation-with-deps", @@ -125,16 +125,6 @@ VERSIONED_CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 8f70e937b..651751db5 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -168,7 +168,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, realisation_with_deps, "realisation-with-deps", @@ -178,16 +178,6 @@ VERSIONED_CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4beced6d8..c3d133e7a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -209,11 +209,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) .outputName = wantedOutput, }}; newRealisation.signatures.clear(); - if (!drv->type().isFixed()) { - auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store; - newRealisation.dependentRealisations = - drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); - } worker.store.signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 8d0a307be..58f3de2b7 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -86,32 +86,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() if (!outputInfo) continue; - bool failed = false; - Goals waitees; - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->config.getHumanReadableURI(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath)); - failed = true; - break; - } - waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId)); - } - } - - if (failed) - continue; - waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); co_await await(std::move(waitees)); diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c5e4e3897..d563b33d8 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -12,30 +12,3 @@ create table if not exists Realisations ( ); create index if not exists IndexRealisations on Realisations(drvPath, outputName); - --- We can end-up in a weird edge-case where a path depends on itself because --- it’s an output of a CA derivation, that happens to be the same as one of its --- dependencies. --- In that case we have a dependency loop (path -> realisation1 -> realisation2 --- -> path) that we need to break by removing the dependencies between the --- realisations -create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict -); --- used by deletion trigger -create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); - --- used by QueryRealisationReferences -create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); --- used by cascade deletion when ValidPaths is deleted -create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index e8a71862e..45ebe8c92 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -56,14 +56,6 @@ struct UnkeyedRealisation StringSet signatures; - /** - * The realisations that are required for the current one to be valid. - * - * When importing this realisation, the store will first check that all its - * dependencies exist, and map to the correct output path - */ - std::map dependentRealisations; - std::string fingerprint(const DrvOutput & key) const; void sign(const DrvOutput & key, const Signer &); @@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation bool isCompatibleWith(const UnkeyedRealisation & other) const; - static std::set closure(Store &, const std::set &); - - static void closure(Store &, const std::set &, std::set & res); - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -154,10 +142,6 @@ struct RealisedPath */ const StorePath & path() const &; - void closure(Store & store, Set & ret) const; - static void closure(Store & store, const Set & startPaths, Set & ret); - Set closure(Store & store) const; - bool operator==(const RealisedPath &) const = default; auto operator<=>(const RealisedPath &) const = default; }; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index d03e8e010..9f445bf1d 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -1006,7 +1006,4 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr); - } // namespace nix diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 59d5cc24f..a068af4fe 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -390,21 +390,6 @@ LocalStore::LocalStore(ref config) where drvPath = ? ; )"); - state->stmts->QueryRealisationReferences.create( - state->db, - R"( - select drvPath, outputName from Realisations - join RealisationsRefs on realisationReference = Realisations.id - where referrer = ?; - )"); - state->stmts->AddRealisationReference.create( - state->db, - R"( - insert or replace into RealisationsRefs (referrer, realisationReference) - values ( - (select id from Realisations where drvPath = ? and outputName = ?), - (select id from Realisations where drvPath = ? and outputName = ?)); - )"); } } @@ -654,25 +639,6 @@ void LocalStore::registerDrvOutput(const Realisation & info) concatStringsSep(" ", info.signatures)) .exec(); } - for (auto & [outputId, depPath] : info.dependentRealisations) { - auto localRealisation = queryRealisationCore_(*state, outputId); - if (!localRealisation) - throw Error( - "unable to register the derivation '%s' as it " - "depends on the non existent '%s'", - info.id.to_string(), - outputId.to_string()); - if (localRealisation->second.outPath != depPath) - throw Error( - "unable to register the derivation '%s' as it " - "depends on a realisation of '%s' that doesn’t" - "match what we have locally", - info.id.to_string(), - outputId.to_string()); - state->stmts->AddRealisationReference - .use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName) - .exec(); - } }); } @@ -1611,21 +1577,6 @@ std::optional LocalStore::queryRealisation_(LocalStore return std::nullopt; auto [realisationDbId, res] = *maybeCore; - std::map dependentRealisations; - auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId)); - while (useRealisationRefs.next()) { - auto depId = DrvOutput{ - Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), - useRealisationRefs.getStr(1), - }; - auto dependentRealisation = queryRealisationCore_(state, depId); - assert(dependentRealisation); // Enforced by the db schema - auto outputPath = dependentRealisation->second.outPath; - dependentRealisations.insert({depId, outputPath}); - } - - res.dependentRealisations = dependentRealisations; - return {res}; } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a31d149c2..d52b23f70 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -329,65 +329,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } -std::map -drvOutputReferences(const std::set & inputRealisations, const StorePathSet & pathReferences) -{ - std::map res; - - for (const auto & input : inputRealisations) { - if (pathReferences.count(input.outPath)) { - res.insert({input.id, input.outPath}); - } - } - - return res; -} - -std::map -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_) -{ - auto & evalStore = evalStore_ ? *evalStore_ : store; - - std::set inputRealisations; - - std::function::ChildNode &)> accumRealisations; - - accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) { - auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv)); - for (const auto & outputName : inputNode.value) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv)); - DrvOutput key{*outputHash, outputName}; - auto thisRealisation = store.queryRealisation(key); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv)); - inputRealisations.insert({*thisRealisation, std::move(key)}); - } - } - if (!inputNode.value.empty()) { - auto d = makeConstantStorePathRef(inputDrv); - for (const auto & [outputName, childNode] : inputNode.childMap) { - SingleDerivedPath next = SingleDerivedPath::Built{d, outputName}; - accumRealisations( - // TODO deep resolutions for dynamic derivations, issue #8947, would go here. - resolveDerivedPath(store, next, evalStore_), - childNode); - } - } - }; - - for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) - accumRealisations(inputDrv, inputNode); - - auto info = store.queryPathInfo(outputPath); - - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); -} - OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index a7f3b98d6..326b93bb8 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,6 +1,5 @@ #include "nix/store/realisation.hh" #include "nix/store/store-api.hh" -#include "nix/util/closure.hh" #include "nix/util/signature/local-keys.hh" #include "nix/util/json-utils.hh" #include @@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const return strHash() + "!" + outputName; } -std::set Realisation::closure(Store & store, const std::set & startOutputs) -{ - std::set res; - Realisation::closure(store, startOutputs, res); - return res; -} - -void Realisation::closure(Store & store, const std::set & startOutputs, std::set & res) -{ - auto getDeps = [&](const Realisation & current) -> std::set { - std::set res; - for (auto & [currentDep, _] : current.dependentRealisations) { - if (auto currentRealisation = store.queryRealisation(currentDep)) - res.insert({*currentRealisation, currentDep}); - else - throw Error("Unrealised derivation '%s'", currentDep.to_string()); - } - return res; - }; - - computeClosure( - startOutputs, - res, - [&](const Realisation & current, std::function> &)> processEdges) { - std::promise> promise; - try { - auto res = getDeps(current); - promise.set_value(res); - } catch (...) { - promise.set_exception(std::current_exception()); - } - return processEdges(promise); - }); -} - std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { nlohmann::json serialized = Realisation{*this, key}; @@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const & bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const { - if (outPath == other.outPath) { - if (dependentRealisations.empty() != other.dependentRealisations.empty()) { - warn( - "Encountered a realisation for '%s' with an empty set of " - "dependencies. This is likely an artifact from an older Nix. " - "I’ll try to fix the realisation if I can", - id.to_string()); - return true; - } else if (dependentRealisations == other.dependentRealisations) { - return true; - } - } - return false; -} - -void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret) -{ - // FIXME: This only builds the store-path closure, not the real realisation - // closure - StorePathSet initialStorePaths, pathsClosure; - for (auto & path : startPaths) - initialStorePaths.insert(path.path()); - store.computeFSClosure(initialStorePaths, pathsClosure); - ret.insert(startPaths.begin(), startPaths.end()); - ret.insert(pathsClosure.begin(), pathsClosure.end()); -} - -void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const -{ - RealisedPath::closure(store, {*this}, ret); -} - -RealisedPath::Set RealisedPath::closure(Store & store) const -{ - RealisedPath::Set ret; - closure(store, ret); - return ret; + return outPath == other.outPath; } } // namespace nix @@ -152,27 +80,19 @@ UnkeyedRealisation adl_serializer::from_json(const json & js if (auto signaturesOpt = optionalValueAt(json, "signatures")) signatures = *signaturesOpt; - std::map dependentRealisations; - if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations")) - for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies)) - dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath}); - return UnkeyedRealisation{ .outPath = valueAt(json, "outPath"), .signatures = signatures, - .dependentRealisations = dependentRealisations, }; } void adl_serializer::to_json(json & json, const UnkeyedRealisation & r) { - auto jsonDependentRealisations = nlohmann::json::object(); - for (auto & [depId, depOutPath] : r.dependentRealisations) - jsonDependentRealisations.emplace(depId.to_string(), depOutPath); json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - {"dependentRealisations", jsonDependentRealisations}, + // back-compat + {"dependentRealisations", json::object()}, }; } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 5270f7d10..91c294987 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -292,7 +292,7 @@ std::vector RestrictedStore::buildPathsWithResults( next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); - for (auto & real : Realisation::closure(*next, newRealisations)) + for (auto & real : newRealisations) goal.addedDrvOutputs.insert(real.id); return results; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cdca6a763..7ec4bc3a9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -908,36 +908,21 @@ std::map copyPaths( SubstituteFlag substitute) { StorePathSet storePaths; - std::set toplevelRealisations; + std::vector realisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto * realisation = std::get_if(&path.raw)) { experimentalFeatureSettings.require(Xp::CaDerivations); - toplevelRealisations.insert(*realisation); + realisations.push_back(realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { - // Copy the realisation closure - processGraph( - Realisation::closure(srcStore, toplevelRealisations), - [&](const Realisation & current) -> std::set { - std::set children; - for (const auto & [drvOutput, _] : current.dependentRealisations) { - auto currentChild = srcStore.queryRealisation(drvOutput); - if (!currentChild) - throw Error( - "incomplete realisation closure: '%s' is a " - "dependency of '%s' but isn't registered", - drvOutput.to_string(), - current.id.to_string()); - children.insert({*currentChild, drvOutput}); - } - return children; - }, - [&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); }); + // Copy the realisations. TODO batch this + for (const auto * realisation : realisations) + dstStore.registerDrvOutput(*realisation, checkSigs); } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want @@ -1048,8 +1033,19 @@ void copyClosure( if (&srcStore == &dstStore) return; - RealisedPath::Set closure; - RealisedPath::closure(srcStore, paths, closure); + StorePathSet closure0; + for (auto & path : paths) { + if (auto * opaquePath = std::get_if(&path.raw)) { + closure0.insert(opaquePath->path); + } + } + + StorePathSet closure1; + srcStore.computeFSClosure(closure0, closure1); + + RealisedPath::Set closure = paths; + for (auto && path : closure1) + closure.insert({std::move(path)}); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index 4a5e8c042..032fb6164 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -25,4 +25,9 @@ nix build -f nondeterministic.nix dep2 --no-link # If everything goes right, we should rebuild dep2 rather than fetch it from # the cache (because that would mean duplicating `current-time` in the closure), # and have `dep1 == dep2`. + +# FIXME: Force the use of small-step resolutions only to fix this in a +# better way (#11896, #11928). +skipTest "temporarily broken because dependent realisations are removed" + nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 9728470f0..2f6ebcef5 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -22,7 +22,10 @@ nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA +# FIXME: `dependentCA` should not need to be explicitly mentioned in +# this. Force the use of small-step resolutions only to allow not +# mentioning it explicitly again. (#11896, #11928). +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA dependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too From 043eed85da1ecfd1936fb56e569eee6a08b9b591 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Feb 2025 23:54:24 -0500 Subject: [PATCH 08/13] Revert "Use the hash modulo in the derivation outputs" Fix #11897 As described in the issue, this makes for a simpler and much more intuitive notion of a realisation key. This is better for pedagogy, and interoperability between more tools. The way the issue was written was that we would switch to only having shallow realisations first, and then do this. But going to only shallow realisations is more complex change, and it turns out we weren't even testing for the benefits that derivation hashes (modulo FODs) provided in the deep realisation case, so I now just want to do this first. Doing this gets the binary cache data structures in order, which will unblock the Hydra fixed-output-derivation tracking work. I don't want to delay that work while I figure out the changes needed for shallow-realisations only. This reverts commit bab1cda0e6c30e25460b5a9c809589d3948f35df. --- src/libcmd/built-path.cc | 18 +- src/libexpr/primops.cc | 23 +- src/libstore-tests/common-protocol.cc | 55 ----- .../data/realisation/simple.json | 12 +- .../data/realisation/with-signature.json | 16 +- .../data/serve-protocol/build-result-2.8.bin | Bin 0 -> 360 bytes .../data/serve-protocol/drv-output-2.8.bin | Bin 0 -> 160 bytes .../data/serve-protocol/realisation-2.8.bin | Bin 0 -> 176 bytes .../unkeyed-realisation-2.8.bin | Bin 0 -> 96 bytes .../worker-protocol/build-result-1.39.bin | Bin 0 -> 424 bytes .../data/worker-protocol/drv-output-1.39.bin | Bin 0 -> 160 bytes .../data/worker-protocol/realisation-1.39.bin | Bin 0 -> 176 bytes .../unkeyed-realisation-1.39.bin | Bin 0 -> 96 bytes src/libstore-tests/dummy-store.cc | 11 +- src/libstore-tests/realisation.cc | 65 ++--- src/libstore-tests/serve-protocol.cc | 140 ++++++----- src/libstore-tests/worker-protocol.cc | 173 +++++++------ src/libstore/binary-cache-store.cc | 12 +- .../build/derivation-building-goal.cc | 7 +- src/libstore/build/derivation-goal.cc | 47 +--- .../build/drv-output-substitution-goal.cc | 7 +- src/libstore/common-protocol.cc | 28 --- src/libstore/daemon.cc | 26 +- src/libstore/derivations.cc | 231 +++++++++++------- src/libstore/dummy-store.cc | 4 +- .../include/nix/store/binary-cache-store.hh | 9 +- .../store/build/derivation-building-misc.hh | 1 - .../nix/store/build/derivation-goal.hh | 2 - .../include/nix/store/common-protocol-impl.hh | 5 +- .../include/nix/store/common-protocol.hh | 7 +- src/libstore/include/nix/store/derivations.hh | 61 +++-- .../include/nix/store/dummy-store-impl.hh | 2 +- .../store/length-prefixed-protocol-helper.hh | 20 +- src/libstore/include/nix/store/realisation.hh | 75 +++--- .../include/nix/store/serve-protocol-impl.hh | 6 +- .../include/nix/store/serve-protocol.hh | 15 +- .../include/nix/store/worker-protocol-impl.hh | 6 +- .../include/nix/store/worker-protocol.hh | 17 +- src/libstore/local-binary-cache-store.cc | 5 +- src/libstore/local-store.cc | 8 +- src/libstore/misc.cc | 9 +- src/libstore/nar-info-disk-cache.cc | 49 ++-- src/libstore/realisation.cc | 91 ++++--- src/libstore/remote-store.cc | 44 +--- src/libstore/restricted-store.cc | 15 +- src/libstore/serve-protocol.cc | 109 ++++++++- src/libstore/store-api.cc | 7 +- src/libstore/unix/build/derivation-builder.cc | 5 +- src/libstore/worker-protocol.cc | 144 ++++++++++- src/nix/build-remote/build-remote.cc | 6 +- src/nix/develop.cc | 10 +- src/perl/lib/Nix/Store.xs | 7 +- tests/functional/ca/substitute.sh | 6 +- 53 files changed, 918 insertions(+), 698 deletions(-) create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/drv-output-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/drv-output-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fc7f18493..f60e4499c 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -108,20 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { - auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath->outPath()), - outputName); - DrvOutput key{*drvOutput, outputName}; + DrvOutput key{ + .drvPath = p.drvPath->outPath(), + .outputName = outputName, + }; auto thisRealisation = store.queryRealisation(key); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(Realisation{*thisRealisation, std::move(key)}); + // We’ve built it, so we must have the realisation. + assert(thisRealisation); + res.insert(Realisation{*thisRealisation, key}); } else { res.insert(outputPath); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5f06bf009..3970a77fd 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1718,28 +1718,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{}); } - auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); - switch (hashModulo.kind) { - case DrvHash::Kind::Regular: - for (auto & i : outputs) { - auto h = get(hashModulo.hashes, i); - if (!h) - state.error("derivation produced no hash for output '%s'", i).atPos(v).debugThrow(); - auto outPath = state.store->makeOutputPath(i, *h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign( - i, - DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }); - } - break; - ; - case DrvHash::Kind::Deferred: - for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{}); - } - } + resolveInputAddressed(*state.store, drv); } /* Write the resulting term into the Nix store directory. */ diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 16030c8f1..4b1d243f9 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -99,61 +99,6 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( - drvOutput, - "drv-output", - (std::tuple{ - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - })) - -CHARACTERIZATION_TEST( - realisation, - "realisation", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - -READ_CHARACTERIZATION_TEST( - realisation_with_deps, - "realisation-with-deps", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 2ccb1e721..1e4760b56 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,6 +1,10 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } } diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index a28848cb0..4a73ed3ef 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,8 +1,12 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [ - "asdfasdfasdf" - ] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdfasdfasdf" + ] + } } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..fa072599578a351e0f289c1c6bbdc5c1ff90107c GIT binary patch literal 360 zcmZQ&fBjf;Y*hsmSsV}eS+ z%uh-z0*mN_Nd^Y}yvz#y;*$KLRQ+_ra`TKz<3e1tE=(^-E6lvK{Cp6XfgJN-dO-#N E0Qb5q`2YX_ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5be0b15a34567aed217e56dbf2543f2f60a200be GIT binary patch literal 160 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<$PJj|FTB M14ChHX$6Q00QuS{z5oCK literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5295ee34400a58e06c919eaf4b0c70b95b713fc9 GIT binary patch literal 176 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<-kNBm)D9 Z<}olq^|3(d#Nw1R5EI5PEKe;0@c}{FC(r-@ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..10f4ebcb2eb1622e1c56b462f36a7db1e863b345 GIT binary patch literal 96 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6GR&W3zSYQPDukXVf@1K)FKcc E0Dd79Jpcdz literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.39.bin b/src/libstore-tests/data/worker-protocol/build-result-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..11bec3e6e3b7a82f7b2d792d687a807644ff9d05 GIT binary patch literal 424 zcmZQ&fBO)Le|4^xkB4qQI< ztSm?kobS(|0^&1)2nGfQn0t~Ei@+lKV3L7BKQFUFzqlm7C{;h*u-rVO(zp;8tqao& U(h4&#Ek7T`Wgy2qm|lN>LeDBQuy}U`R@=0<$PJj|FTB M14ChHX$6Q00QuS{z5oCK literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation-1.39.bin b/src/libstore-tests/data/worker-protocol/realisation-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..5295ee34400a58e06c919eaf4b0c70b95b713fc9 GIT binary patch literal 176 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<-kNBm)D9 Z<}olq^|3(d#Nw1R5EI5PEKe;0@c}{FC(r-@ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..10f4ebcb2eb1622e1c56b462f36a7db1e863b345 GIT binary patch literal 96 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6GR&W3zSYQPDukXVf@1K)FKcc E0Dd79Jpcdz literal 0 HcmV?d00001 diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 3dd8137a3..49ffbf75e 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -16,20 +16,19 @@ TEST(DummyStore, realisation_read) return cfg->openDummyStore(); }(); - auto drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }; - store->buildTrace.insert({drvHash, {{outputName, make_ref(value)}}}); + store->buildTrace.insert({drvPath, {{outputName, make_ref(value)}}}); - auto value2 = store->queryRealisation({drvHash, outputName}); + auto value2 = store->queryRealisation({drvPath, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 66f4707e9..3087bd1b5 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -44,45 +44,30 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } -Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, -}; - -INSTANTIATE_TEST_SUITE_P( - RealisationJSON, - RealisationJsonTest, - ::testing::Values( - std::pair{ - "simple", - simple, - }, - std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = {"asdfasdfasdf"}; - return r; - }(), - })); - -/** - * We no longer have a notion of "dependent realisations", but we still - * want to parse old realisation files. So make this just be a read test - * (no write direction), accordingly. - */ -TEST_F(RealisationTest, dependent_realisations_from_json) -{ - readJsonTest("with-dependent-realisations", simple); -} +INSTANTIATE_TEST_SUITE_P(RealisationJSON, RealisationJsonTest, ([] { + Realisation simple{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", + }, + }; + return ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = {"asdfasdfasdf"}; + return r; + }(), + }); + }())); } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 0821bb99b..b0dfb191c 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -72,16 +72,16 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput, - "drv-output", - defaultVersion, + drvOutput_2_8, + "drv-output-2.8", + 2 << 8 | 8, (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) @@ -90,46 +90,27 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_2_8, + "unkeyed-realisation-2.8", + 2 << 8 | 8, + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_2_8, + "realisation-2.8", + 2 << 8 | 8, + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -179,7 +160,10 @@ VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2 t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully right + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -205,27 +189,65 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, buildResult_2_8, "build-result-2.8", 2 << 8 | 8, ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + }}}, + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 651751db5..761444119 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -128,61 +128,42 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output", - defaultVersion, + "drv-output-1.39", + 1 << 8 | 39, (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_1_39, + "unkeyed-realisation-1.39", + 1 << 8 | 39, + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_1_39, + "realisation-1.39", + 1 << 8 | 39, + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -204,7 +185,10 @@ VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully right + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -223,25 +207,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -250,7 +222,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -276,27 +249,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -309,7 +268,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -335,27 +295,60 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, buildResult_1_39, "build-result-1.39", 1 << 8 | 39, ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + }}}, + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3705f3d4d..7b686fddd 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -504,7 +504,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.to_string() + ".doi"; + return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -525,7 +525,10 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); + {}, + "while parsing file '%s' as a build trace value for key '%s'", + outputInfoFilePath, + id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -541,7 +544,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); + upsertFile( + makeRealisationPath(info.id), + static_cast(static_cast(info)).dump(), + "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 68a86cac2..c1f87e381 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -212,9 +212,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild() given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; + for (auto & [outputName, _] : drv->outputs) { + InitialOutput v; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -1096,7 +1095,7 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{info.outputHash, i.first}; + auto drvOutput = DrvOutput{drvPath, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c3d133e7a..f5f74d6de 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -36,12 +36,6 @@ DerivationGoal::DerivationGoal( , drvPath(drvPath) , wantedOutput(wantedOutput) , drv{std::make_unique(drv)} - , outputHash{[&] { - auto outputHashes = staticOutputHashes(worker.evalStore, drv); - if (auto * mOutputHash = get(outputHashes, wantedOutput)) - return *mOutputHash; - throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); - }()} , buildMode(buildMode) { @@ -100,7 +94,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (settings.useSubstitutes && drvOptions.substitutesAllowed()) { if (!checkResult) - waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput}))); + waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{drvPath, wantedOutput}))); else { auto * cap = getDerivationCA(*drv); waitees.insert(upcast_goal(worker.makePathSubstitutionGoal( @@ -167,12 +161,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); - - auto outputHash = get(outputHashes, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); - if ((!outputHash) || (!resolvedHash)) + if (!drv->outputs.contains(wantedOutput)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -181,7 +170,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return static_cast(*take1); + return *take1; /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -189,7 +178,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvHash = *resolvedHash, + .drvPath = pathResolved, .outputName = wantedOutput, }); if (take2) @@ -205,7 +194,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) Realisation newRealisation{ realisation, { - .drvHash = *outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }}; newRealisation.signatures.clear(); @@ -251,16 +240,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -369,7 +349,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{outputHash, wantedOutput}; + auto drvOutput = DrvOutput{drvPath, wantedOutput}; std::optional mRealisation; @@ -409,7 +389,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvHash = outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }, }); @@ -432,16 +412,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke { buildResult.inner = BuildResult::Success{ .status = status, - .builtOutputs = {{ - wantedOutput, - { - std::move(builtOutput), - DrvOutput{ - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, }; mcExpectedBuilds.reset(); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 58f3de2b7..34a263edd 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -12,7 +12,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.to_string()); + name = fmt("substitution of '%s'", id.render(worker.store)); trace("created"); } @@ -107,7 +107,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug( + "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); if (substituterFailed) { worker.failedSubstitutions++; @@ -122,7 +123,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.to_string()); + return "a$" + std::string(id.render(worker.store)); } void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index b069c9498..6b819fd6e 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -46,34 +46,6 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - try { - return nlohmann::json::parse(rawInput); - } catch (Error & e) { - e.addTrace({}, "while parsing a realisation object in the remote protocol"); - throw; - } -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << static_cast(realisation).dump(); -} - -DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6d2a5781..50614d3f5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -963,33 +963,31 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { - auto outputId = DrvOutput::parse(readString(conn.from)); - auto outputPath = StorePath(readString(conn.from)); - store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); - } else { - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); - } + // TODO move to WorkerProto::Serialise and friends + // if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) { + // throw Error("old-style build traces no longer supported"); + //} + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); logger->stopWork(); break; } case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(conn.from)); - auto info = store->queryRealisation(outputId); + auto outputId = WorkerProto::Serialise::read(*store, rconn); + std::optional info = *store->queryRealisation(outputId); logger->stopWork(); if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); WorkerProto::write(*store, wconn, outPaths); + } else if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) { + // No longer support this format + WorkerProto::write(*store, wconn, StringSet{}); } else { - std::set realisations; - if (info) - realisations.insert({*info, outputId}); - WorkerProto::write(*store, wconn, realisations); + WorkerProto::write(*store, wconn, info); } break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f44bf3e70..e4c42339d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -847,13 +847,14 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) +static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); + // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -876,12 +877,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto type = drv.type(); - /* Return a fixed hash for fixed-output derivations. */ - if (type.isFixed()) { + if (drv.type().isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -891,54 +890,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return DrvHash{ - .hashes = outputHashes, - .kind = DrvHash::Kind::Regular, - }; + return outputHashes; } - auto kind = std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return DrvHash::Kind::Regular; - }, - [](const DerivationType::ContentAddressed & ca) { - return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; - }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, - drv.type().raw); + if (std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return false; + }, + [](const DerivationType::ContentAddressed & ca) { + // Already covered + assert(!ca.fixed); + return true; + }, + [](const DerivationType::Impure &) { return true; }}, + drv.type().raw)) { + return DrvHashModulo::DeferredDrv{}; + } + /* For other derivations, replace the inputs paths with recursive + calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { + /* Need to build and resolve dynamic derivations first */ + if (!node.childMap.empty()) { + return DrvHashModulo::DeferredDrv{}; + } + const auto & res = pathDerivationModulo(store, drvPath); - if (res.kind == DrvHash::Kind::Deferred) - kind = DrvHash::Kind::Deferred; - for (auto & outputName : node.value) { - const auto h = get(res.hashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); + if (std::visit( + overloaded{ + [&](const DrvHashModulo::DeferredDrv &) { return true; }, + // Regular non-CA derivation, replace derivation + [&](const DrvHashModulo::DrvHash & drvHash) { + inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + return false; + }, + // CA derivation's output hashes + [&](const DrvHashModulo::CaOutputHashes & outputHashes) { + for (auto & outputName : node.value) { + /* Put each one in with a single "out" output.. */ + const auto h = get(outputHashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2.insert_or_assign( + h->to_string(HashFormat::Base16, false), + DerivedPathMap::ChildNode{ + .value = {"out"}, + }); + } + return false; + }, + }, + res.raw)) { + return DrvHashModulo::DeferredDrv{}; } } - auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); - - std::map outputHashes; - for (const auto & [outputName, _] : drv.outputs) { - outputHashes.insert_or_assign(outputName, hash); - } - - return DrvHash{ - .hashes = outputHashes, - .kind = kind, - }; -} - -std::map staticOutputHashes(Store & store, const Derivation & drv) -{ - return hashDerivationModulo(store, drv, true).hashes; + return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1088,22 +1099,39 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +void resolveInputAddressed(Store & store, Derivation & drv) { - drv.applyRewrites(rewrites); + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, drv, true); + } + return *hashModulo_; + }; - auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative(output.raw)) { - auto h = get(hashModulo.hashes, outputName); - if (!h) - throw Error( - "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); - auto outPath = store.makeOutputPath(outputName, *h, drv.name); - drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }; + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + auto outPath = store.makeOutputPath(outputName, drvHash, drv.name); + drv.env.insert_or_assign(outputName, store.printStorePath(outPath)); + output = DerivationOutput::InputAddressed{ + .path = std::move(outPath), + }; + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + deferred (waiting to be input-addressed). */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + // Nothing to do, already deferred + }, + }, + hashModulo().raw); } } } @@ -1186,9 +1214,13 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - rewriteDerivation(store, resolved, inputRewrites); + resolved.applyRewrites(inputRewrites); - return resolved; + Derivation resolved2{std::move(resolved)}; + + resolveInputAddressed(store, resolved2); + + return resolved2; } void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const @@ -1216,46 +1248,79 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const // combinations that are currently prohibited. type(); - std::optional hashesModulo; - for (auto & i : outputs) { + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, *this, true); + } + return *hashModulo_; + }; + + for (auto & [outputName, output] : outputs) { std::visit( overloaded{ [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(store, *this, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error( - "derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - store.printStorePath(drvPath), - store.printStorePath(doia.path), - i.first); - StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error( - "derivation '%s' has incorrect output '%s', should be '%s'", - store.printStorePath(drvPath), - store.printStorePath(doia.path), - store.printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + StorePath recomputed = store.makeOutputPath(outputName, drvHash, drvName); + if (doia.path != recomputed) + throw Error( + "derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), + store.printStorePath(doia.path), + store.printStorePath(recomputed)); + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed. */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + throw Error( + "derivation '%s' has output '%s', but derivation is not yet ready to be input-addressed", + store.printStorePath(drvPath), + store.printStorePath(doia.path)); + }, + }, + hashModulo().raw); + envHasRightPath(doia.path, outputName); }, [&](const DerivationOutput::CAFixed & dof) { - auto path = dof.path(store, drvName, i.first); - envHasRightPath(path, i.first); + auto path = dof.path(store, drvName, outputName); + envHasRightPath(path, outputName); }, [&](const DerivationOutput::CAFloating &) { /* Nothing to check */ }, [&](const DerivationOutput::Deferred &) { /* Nothing to check */ + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + throw Error( + "derivation '%s' has deferred output '%s', yet is ready to be input-addressed", + store.printStorePath(drvPath), + outputName); + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed. */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + /* Nothing to check */ + }, + }, + hashModulo().raw); }, [&](const DerivationOutput::Impure &) { /* Nothing to check */ }, }, - i.second.raw); + output.raw); } } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 509b7a0b1..8aabb64fb 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -253,7 +253,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { auto ref = make_ref(output); - buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, ref}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, ref}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, make_ref(output)); }); } @@ -274,7 +274,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(it->second.get_ptr()); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 3f4de2bd4..ddac2de17 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -82,8 +82,13 @@ protected: /** * The prefix under which realisation infos will be stored + * + * @note The previous (still experimental, though) hash-keyed + * realisations were under "realisations". "build trace" is a better + * name anyways (issue #11895), and this serves as some light + * versioning. */ - constexpr const static std::string realisationsPrefix = "realisations"; + constexpr const static std::string realisationsPrefix = "build-trace"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; @@ -92,7 +97,7 @@ protected: /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvOutput}.doi`. + * It's `${realisationsPrefix}/${drvPath}/${outputName}`. */ std::string makeRealisationPath(const DrvOutput & id); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 2b68fa178..8d6892839 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,7 +45,6 @@ struct InitialOutputStatus struct InitialOutput { - Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 0fe610987..c732fd441 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -71,8 +71,6 @@ private: */ std::unique_ptr drv; - const Hash outputHash; - const BuildMode buildMode; /** diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index cb1020a3c..d0eebe0bc 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,12 +26,13 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index c1d22fa6c..45bec3ff2 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -12,7 +12,6 @@ struct Source; class StorePath; struct ContentAddress; struct DrvOutput; -struct Realisation; /** * Shared serializers between the worker protocol, serve protocol, and a @@ -70,8 +69,6 @@ template<> DECLARE_COMMON_SERIALISER(ContentAddress); template<> DECLARE_COMMON_SERIALISER(DrvOutput); -template<> -DECLARE_COMMON_SERIALISER(Realisation); #define COMMA_ , template @@ -81,8 +78,8 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); +template +DECLARE_COMMON_SERIALISER(std::map); #undef COMMA_ /** diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 4615d8acd..89db72c28 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -427,34 +427,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHash +struct DrvHashModulo { /** - * Map from output names to hashes + * Single hash for the derivation + * + * This is for an input-addressed derivation that doesn't + * transitively depend on any floating-CA derivations. */ - std::map hashes; - - enum struct Kind : bool { - /** - * Statically determined derivations. - * This hash will be directly used to compute the output paths - */ - Regular, - - /** - * Floating-output derivations (and their reverse dependencies). - */ - Deferred, - }; + using DrvHash = Hash; /** - * The kind of derivation this is, simplified for just "derivation hash - * modulo" purposes. + * Known CA drv's output hashes, for fixed-output derivations whose + * output hashes are always known since they are fixed up-front. */ - Kind kind; -}; + using CaOutputHashes = std::map; -void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + /** + * This derivation doesn't yet have known output hashes. + * + * Either because itself is floating CA, or it (transtively) depends + * on a floating CA derivation. + */ + using DeferredDrv = std::monostate; + + using Raw = std::variant; + + Raw raw; + + bool operator==(const DrvHashModulo &) const = default; + // auto operator <=> (const DrvHashModulo &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); +}; /** * Returns hashes with the details of fixed-output subderivations @@ -480,15 +485,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * Return a map associating each output to a hash that uniquely identifies its - * derivation (modulo the self-references). + * If a derivation is input addressed and doesn't yet have its input + * addressed (is deferred) try using `hashDerivationModulo`. * - * \todo What is the Hash in this map? + * Does nothing if not deferred input-addressed, or + * `hashDerivationModulo` indicates it is missing inputs' output paths + * and is not yet ready (and must stay deferred). */ -std::map staticOutputHashes(Store & store, const Derivation & drv); +void resolveInputAddressed(Store & store, Derivation & drv); struct DrvHashFct { @@ -503,7 +510,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 4c9f54e98..ec66cb948 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -40,7 +40,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map>> buildTrace; + boost::concurrent_flat_map>> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 035019340..e1a80e8dc 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); -#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); +#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map -LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ LengthPrefixedProtoHelper>::read(const StoreDirConfig & st return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 45ebe8c92..e5ac6d73d 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -18,33 +18,40 @@ struct OutputsSpec; /** * A general `Realisation` key. * - * This is similar to a `DerivedPath::Opaque`, but the derivation is - * identified by its "hash modulo" instead of by its store path. + * This is similar to a `DerivedPath::Built`, except it is only a single + * step: `drvPath` is a `StorePath` rather than a `DerivedPath`. */ struct DrvOutput { /** - * The hash modulo of the derivation. - * - * Computed from the derivation itself for most types of - * derivations, but computed from the (fixed) content address of the - * output for fixed-output derivations. + * The store path to the derivation */ - Hash drvHash; + StorePath drvPath; /** * The name of the output. */ OutputName outputName; + /** + * Skips the store dir on the `drvPath` + */ std::string to_string() const; - std::string strHash() const - { - return drvHash.to_string(HashFormat::Base16, true); - } + /** + * Skips the store dir on the `drvPath` + */ + static DrvOutput from_string(std::string_view); - static DrvOutput parse(const std::string &); + /** + * Includes the store dir on `drvPath` + */ + std::string render(const StoreDirConfig & store) const; + + /** + * Includes the store dir on `drvPath` + */ + static DrvOutput parse(const StoreDirConfig & store, std::string_view); bool operator==(const DrvOutput &) const = default; auto operator<=>(const DrvOutput &) const = default; @@ -64,6 +71,16 @@ struct UnkeyedRealisation size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; + /** + * Just check the `outPath`. Signatures don't matter for this. + * Callers must ensure that the corresponding key is the same for + * most use-cases. + */ + bool isCompatibleWith(const UnkeyedRealisation & other) const + { + return outPath == other.outPath; + } + const StorePath & getPath() const { return outPath; @@ -77,8 +94,6 @@ struct Realisation : UnkeyedRealisation { DrvOutput id; - bool isCompatibleWith(const UnkeyedRealisation & other) const; - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -89,16 +104,7 @@ struct Realisation : UnkeyedRealisation * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; - -/** - * Collection type for multiple derivations' outputs' `Realisation`s. - * - * `DrvOutput` is used because in general the derivations are not all - * the same, so we need to identify firstly which derivation, and - * secondly which output of that derivation. - */ -typedef std::map DrvOutputs; +typedef std::map SingleDrvOutputs; struct OpaquePath { @@ -149,22 +155,21 @@ struct RealisedPath class MissingRealisation : public Error { public: - MissingRealisation(DrvOutput & outputId) - : MissingRealisation(outputId.outputName, outputId.strHash()) + MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId) + : MissingRealisation(store, outputId.drvPath, outputId.outputName) { } - MissingRealisation(std::string_view drv, OutputName outputName) - : Error( - "cannot operate on output '%s' of the " - "unbuilt derivation '%s'", - outputName, - drv) - { - } + MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName); + MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName); }; } // namespace nix +JSON_IMPL(nix::DrvOutput) JSON_IMPL(nix::UnkeyedRealisation) JSON_IMPL(nix::Realisation) diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index a9617165a..24fc3c9ab 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -34,8 +34,10 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , SERVE_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index 974bf42d5..1b548b517 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION (2 << 8 | 7) +#define SERVE_PROTOCOL_VERSION (2 << 8 | 8) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,6 +18,9 @@ struct Source; // items being serialised struct BuildResult; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; /** * The "serve protocol", used by ssh:// stores. @@ -178,6 +181,12 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> +DECLARE_SERVE_SERIALISER(DrvOutput); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_SERVE_SERIALISER(Realisation); +template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); @@ -190,8 +199,8 @@ DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -template -DECLARE_SERVE_SERIALISER(std::map); +template +DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 26f6b9d44..605663d2e 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -34,8 +34,10 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , WORKER_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 29d4828c2..c1240f715 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -12,7 +12,7 @@ namespace nix { /* Note: you generally shouldn't change the protocol version. Define a new `WorkerProto::Feature` instead. */ -#define PROTOCOL_VERSION (1 << 8 | 38) +#define PROTOCOL_VERSION (1 << 8 | 39) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -34,6 +34,9 @@ struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; enum BuildMode : uint8_t; enum TrustedFlag : bool; @@ -258,6 +261,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> +DECLARE_WORKER_SERIALISER(DrvOutput); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_WORKER_SERIALISER(Realisation); +template<> +DECLARE_WORKER_SERIALISER(std::optional); +template<> DECLARE_WORKER_SERIALISER(BuildMode); template<> DECLARE_WORKER_SERIALISER(std::optional); @@ -274,8 +285,8 @@ DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -template -DECLARE_WORKER_SERIALISER(std::map); +template +DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b5e43de68..5fcfb0128 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -58,9 +58,10 @@ protected: std::shared_ptr> istream, const std::string & mimeType) override { - auto path2 = config->binaryCacheDir + "/" + path; + auto path2 = std::filesystem::path{config->binaryCacheDir} / path; static std::atomic counter{0}; - Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); + createDirs(path2.parent_path()); + auto tmp = path2 + fmt(".tmp.%d.%d", getpid(), ++counter); AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a068af4fe..bf058bedd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -110,8 +110,6 @@ struct LocalStore::State::Stmts SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; - SQLiteStmt QueryRealisationReferences; - SQLiteStmt AddRealisationReference; }; LocalStore::LocalStore(ref config) @@ -621,7 +619,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto combinedSignatures = oldR->signatures; combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput - .use()(concatStringsSep(" ", combinedSignatures))(info.id.strHash())(info.id.outputName) + .use()(concatStringsSep(" ", combinedSignatures))(info.id.drvPath.to_string())(info.id.outputName) .exec(); } else { throw Error( @@ -635,7 +633,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) } } else { state->stmts->RegisterRealisedOutput - .use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( + .use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))( concatStringsSep(" ", info.signatures)) .exec(); } @@ -1555,7 +1553,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si std::optional> LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id) { - auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName)); + auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName)); if (!useQueryRealisedOutput.next()) return std::nullopt; auto realisationDbId = useQueryRealisedOutput.getInt(0); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index d52b23f70..463a899ab 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -239,16 +239,15 @@ MissingPaths Store::queryMissing(const std::vector & targets) // If there are unknown output paths, attempt to find if the // paths are known to substituters through a realisation. - auto outputHashes = staticOutputHashes(*this, *drv); knownOutputPaths = true; - for (auto [outputName, hash] : outputHashes) { + for (auto & [outputName, _] : drv->outputs) { if (!bfd.outputs.contains(outputName)) continue; bool found = false; for (auto & sub : getDefaultSubstituters()) { - auto realisation = sub->queryRealisation({hash, outputName}); + auto realisation = sub->queryRealisation({drvPath, outputName}); if (!realisation) continue; found = true; @@ -361,7 +360,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } @@ -385,7 +384,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); + throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output); return *optPath; }, }, diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 11608a667..116957e55 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -44,10 +44,16 @@ create table if not exists NARs ( create table if not exists Realisations ( cache integer not null, - outputId text not null, - content blob, -- Json serialisation of the realisation, or null if the realisation is absent + + drvPath text not null, + outputName text not null, + + -- The following are null if the realisation is absent + outputPath text, + sigs text, + timestamp integer not null, - primary key (cache, outputId), + primary key (cache, drvPath, outputName), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -121,24 +127,24 @@ public: state->insertRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, content, timestamp) - values (?, ?, ?, ?) + insert or replace into Realisations(cache, drvPath, outputName, outputPath, sigs, timestamp) + values (?, ?, ?, ?, ?, ?) )"); state->insertMissingRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, timestamp) - values (?, ?, ?) + insert or replace into Realisations(cache, drvPath, outputName, timestamp) + values (?, ?, ?, ?) )"); state->queryRealisation.create( state->db, R"( - select content from Realisations - where cache = ? and outputId = ? and - ((content is null and timestamp > ?) or - (content is not null and timestamp > ?)) + select outputPath, sigs from Realisations + where cache = ? and drvPath = ? and outputName = ? and + ((outputPath is null and timestamp > ?) or + (outputPath is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -295,22 +301,27 @@ public: auto now = time(0); - auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( + auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)( now - settings.ttlNegativeNarInfoCache)(now - settings.ttlPositiveNarInfoCache)); if (!queryRealisation.next()) - return {oUnknown, 0}; + return {oUnknown, nullptr}; if (queryRealisation.isNull(0)) - return {oInvalid, 0}; + return {oInvalid, nullptr}; try { return { oValid, - std::make_shared(nlohmann::json::parse(queryRealisation.getStr(0))), + std::make_shared( + UnkeyedRealisation{ + .outPath = StorePath{queryRealisation.getStr(0)}, + .signatures = nlohmann::json::parse(queryRealisation.getStr(1)), + }, + id), }; } catch (Error & e) { - e.addTrace({}, "while parsing the local disk cache"); + e.addTrace({}, "reading build trace key-value from the local disk cache"); throw; } }); @@ -355,7 +366,9 @@ public: auto & cache(getCache(*state, uri)); state->insertRealisation - .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())(time(0)) + .use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)( + realisation.outPath.to_string())(static_cast(realisation.signatures).dump())( + time(0)) .exec(); }); } @@ -366,7 +379,7 @@ public: auto state(_state.lock()); auto & cache(getCache(*state, uri)); - state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec(); + state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(0)).exec(); }); } }; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 326b93bb8..591b48fe7 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -8,28 +8,34 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); -DrvOutput DrvOutput::parse(const std::string & strRep) +DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s) { - size_t n = strRep.find("!"); - if (n == strRep.npos) - throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); - + size_t n = s.rfind('^'); + if (n == s.npos) + throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s); return DrvOutput{ - .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), - .outputName = strRep.substr(n + 1), + .drvPath = store.parseStorePath(s.substr(0, n)), + .outputName = OutputName{s.substr(n + 1)}, }; } +std::string DrvOutput::render(const StoreDirConfig & store) const +{ + return std::string(store.printStorePath(drvPath)) + "^" + outputName; +} + std::string DrvOutput::to_string() const { - return strHash() + "!" + outputName; + return std::string(drvPath.to_string()) + "^" + outputName; } std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { - nlohmann::json serialized = Realisation{*this, key}; - serialized.erase("signatures"); - return serialized.dump(); + auto serialised = static_cast(Realisation{*this, key}); + auto value = serialised.find("value"); + assert(value != serialised.end()); + value->erase("signatures"); + return serialised.dump(); } void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) @@ -61,9 +67,20 @@ const StorePath & RealisedPath::path() const & return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw); } -bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName) + : Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath)) { - return outPath == other.outPath; +} + +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName) + : MissingRealisation{store, drvPathResolved, outputName} +{ + addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store)); } } // namespace nix @@ -72,16 +89,34 @@ namespace nlohmann { using namespace nix; -UnkeyedRealisation adl_serializer::from_json(const json & json0) +DrvOutput adl_serializer::from_json(const json & json) { - auto json = getObject(json0); + auto obj = getObject(json); + + return { + .drvPath = valueAt(obj, "drvPath"), + .outputName = getString(valueAt(obj, "outputName")), + }; +} + +void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) +{ + json = { + {"drvPath", drvOutput.drvPath}, + {"outputName", drvOutput.outputName}, + }; +} + +UnkeyedRealisation adl_serializer::from_json(const json & json) +{ + auto obj = getObject(json); StringSet signatures; - if (auto signaturesOpt = optionalValueAt(json, "signatures")) - signatures = *signaturesOpt; + if (auto * signaturesJson = get(obj, "signatures")) + signatures = getStringSet(*signaturesJson); - return UnkeyedRealisation{ - .outPath = valueAt(json, "outPath"), + return { + .outPath = valueAt(obj, "outPath"), .signatures = signatures, }; } @@ -91,25 +126,25 @@ void adl_serializer::to_json(json & json, const UnkeyedReali json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - // back-compat - {"dependentRealisations", json::object()}, }; } -Realisation adl_serializer::from_json(const json & json0) +Realisation adl_serializer::from_json(const nlohmann::json & json) { - auto json = getObject(json0); + auto obj = getObject(json); - return Realisation{ - static_cast(json0), - DrvOutput::parse(valueAt(json, "id")), + return { + static_cast(valueAt(obj, "value")), + static_cast(valueAt(obj, "key")), }; } void adl_serializer::to_json(json & json, const Realisation & r) { - json = static_cast(r); - json["id"] = r.id.to_string(); + json = { + {"key", r.id}, + {"value", static_cast(r)}, + }; } } // namespace nlohmann diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0d83aed4c..c36a39d60 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -507,7 +507,7 @@ void RemoteStore::queryRealisationUncached( try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 39) { warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -516,21 +516,12 @@ void RemoteStore::queryRealisationUncached( conn->to << id.to_string(); conn.processStderr(); - auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { - auto outPaths = WorkerProto::Serialise>::read(*this, *conn); - if (outPaths.empty()) - return nullptr; - return std::make_shared(UnkeyedRealisation{.outPath = *outPaths.begin()}); - } else { - auto realisations = WorkerProto::Serialise>::read(*this, *conn); - if (realisations.empty()) - return nullptr; - return std::make_shared(*realisations.begin()); - } - }(); - - callback(std::shared_ptr(real)); + callback([&]() -> std::shared_ptr { + auto realisation = WorkerProto::Serialise>::read(*this, *conn); + if (!realisation) + return nullptr; + return std::make_shared(*realisation); + }()); } catch (...) { return callback.rethrow(); } @@ -612,30 +603,19 @@ std::vector RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); - auto drv = evalStore->readDerivation(drvPath); - const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - printStorePath(drvPath), - output); - auto outputId = DrvOutput{*outputHash, output}; + auto outputId = DrvOutput{drvPath, output}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw MissingRealisation(outputId); - success.builtOutputs.emplace(output, Realisation{*realisation, outputId}); + throw MissingRealisation(*this, outputId); + success.builtOutputs.emplace(output, *realisation); } else { success.builtOutputs.emplace( output, - Realisation{ - UnkeyedRealisation{ - .outPath = outputPath, - }, - outputId, + UnkeyedRealisation{ + .outPath = outputPath, }); } } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 91c294987..e09fa4dd7 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -281,9 +281,18 @@ std::vector RestrictedStore::buildPathsWithResults( for (auto & result : results) { if (auto * successP = result.tryGetSuccess()) { - for (auto & [outputName, output] : successP->builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert(output); + if (auto * pathBuilt = std::get_if(&result.path)) { + // TODO ugly extra IO + auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath); + for (auto & [outputName, output] : successP->builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert( + {output, + { + .drvPath = drvPath, + .outputName = outputName, + }}); + } } } } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 51b575fcd..d5f8a9ea9 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -6,6 +6,7 @@ #include "nix/store/serve-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include @@ -24,10 +25,19 @@ BuildResult ServeProto::Serialise::read(const StoreDirConfig & stor if (GET_PROTOCOL_MINOR(conn.version) >= 3) conn.from >> status.timesBuilt >> failure.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { - auto builtOutputs = ServeProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (GET_PROTOCOL_MINOR(conn.version) >= 8) { + success.builtOutputs = ServeProto::Serialise>::read(store, conn); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + for (auto & [output, realisation] : ServeProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } if (BuildResult::Success::statusIs(rawStatus)) { @@ -51,15 +61,18 @@ void ServeProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - ServeProto::write(store, conn, builtOutputsFullKey); + + if (GET_PROTOCOL_MINOR(conn.version) >= 8) { + ServeProto::write(store, conn, builtOutputs); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + // We no longer support these types of realisations + ServeProto::write(store, conn, StringMap{}); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -144,4 +157,82 @@ void ServeProto::Serialise::write( } } +UnkeyedRealisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto outPath = ServeProto::Serialise::read(store, conn); + auto signatures = ServeProto::Serialise::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void ServeProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + ServeProto::write(store, conn, info.outPath); + ServeProto::write(store, conn, info.signatures); +} + +DrvOutput ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto drvPath = ServeProto::Serialise::read(store, conn); + auto outputName = ServeProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + ServeProto::write(store, conn, info.drvPath); + ServeProto::write(store, conn, info.outputName); +} + +Realisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = ServeProto::Serialise::read(store, conn); + auto unkeyed = ServeProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + ServeProto::write(store, conn, info.id); + ServeProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7ec4bc3a9..92b036dfe 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -360,9 +360,8 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore return outputs; auto drv = evalStore.readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto & [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); + for (auto & [outputName, _] : drv.outputs) { + auto realisation = queryRealisation(DrvOutput{path, outputName}); if (realisation) { outputs.insert_or_assign(outputName, realisation->outPath); } else { @@ -382,7 +381,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw MissingRealisation(printStorePath(path), outName); + throw MissingRealisation(*this, path, outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index d6abf85e3..c7dd72ed1 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1870,7 +1870,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { .outPath = newInfo.path, }, - DrvOutput{oldinfo->outputHash, outputName}, + DrvOutput{ + .drvPath = drvPath, + .outputName = outputName, + }, }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) { store.signRealisation(thisRealisation); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a17d2c028..c6719974b 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,6 +6,7 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include #include @@ -132,7 +133,7 @@ void WorkerProto::Serialise::write( throw Error( "trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MAJOR(conn.version) >> 8, GET_PROTOCOL_MINOR(conn.version)); }, [&](std::monostate) { @@ -174,14 +175,24 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.from >> res.timesBuilt >> failure.isNonDeterministic >> res.startTime >> res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { res.cpuUser = WorkerProto::Serialise>::read(store, conn); res.cpuSystem = WorkerProto::Serialise>::read(store, conn); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (GET_PROTOCOL_MINOR(conn.version) >= 39) { + success.builtOutputs = WorkerProto::Serialise>::read(store, conn); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + for (auto && [output, realisation] : WorkerProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } if (BuildResult::Success::statusIs(rawStatus)) { @@ -205,20 +216,24 @@ void WorkerProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuSystem); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputsFullKey); + + if (GET_PROTOCOL_MINOR(conn.version) >= 39) { + WorkerProto::write(store, conn, builtOutputs); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + // Don't support those types of realisations anymore. + WorkerProto::write(store, conn, StringMap{}); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -309,4 +324,113 @@ void WorkerProto::Serialise::write( } } +UnkeyedRealisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.39) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto outPath = WorkerProto::Serialise::read(store, conn); + auto signatures = WorkerProto::Serialise::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.39) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + WorkerProto::write(store, conn, info.outPath); + WorkerProto::write(store, conn, info.signatures); +} + +std::optional +WorkerProto::Serialise>::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + // Hack to improve compat + (void) WorkerProto::Serialise::read(store, conn); + return std::nullopt; + } else { + auto temp = readNum(conn.from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return WorkerProto::Serialise::read(store, conn); + default: + throw Error("Invalid optional build trace from remote"); + } + } +} + +void WorkerProto::Serialise>::write( + const StoreDirConfig & store, WriteConn conn, const std::optional & info) +{ + if (!info) { + conn.to << uint8_t{0}; + } else { + conn.to << uint8_t{1}; + WorkerProto::write(store, conn, *info); + } +} + +DrvOutput WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.29) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto drvPath = WorkerProto::Serialise::read(store, conn); + auto outputName = WorkerProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.29) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + WorkerProto::write(store, conn, info.drvPath); + WorkerProto::write(store, conn, info.outputName); +} + +Realisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = WorkerProto::Serialise::read(store, conn); + auto unkeyed = WorkerProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + WorkerProto::write(store, conn, info.id); + WorkerProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index ffb77ddf1..1e52daadc 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -346,13 +346,11 @@ static int main_build_remote(int argc, char ** argv) optResult = std::move(res[0]); } - auto outputHashes = staticOutputHashes(*store, drv); std::set missingRealisations; StorePathSet missingPaths; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { - auto thisOutputHash = outputHashes.at(outputName); - auto thisOutputId = DrvOutput{thisOutputHash, outputName}; + auto thisOutputId = DrvOutput{*drvPath, outputName}; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); assert(optResult); @@ -362,7 +360,7 @@ static int main_build_remote(int argc, char ** argv) auto i = success.builtOutputs.find(outputName); assert(i != success.builtOutputs.end()); auto & newRealisation = i->second; - missingRealisations.insert(newRealisation); + missingRealisations.insert({newRealisation, thisOutputId}); missingPaths.insert(newRealisation.outPath); } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d23dce10b..0f894b9de 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -279,16 +279,8 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore output.second = DerivationOutput::Deferred{}; drv.env[output.first] = ""; } - auto hashesModulo = hashDerivationModulo(*evalStore, drv, true); - for (auto & output : drv.outputs) { - Hash h = hashesModulo.hashes.at(output.first); - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = DerivationOutput::InputAddressed{ - .path = outPath, - }; - drv.env[output.first] = store->printStorePath(outPath); - } + resolveInputAddressed(*evalStore, drv); } auto shellDrvPath = writeDerivation(*evalStore, drv); diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 93e9f0f95..6d5dcb400 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -163,10 +163,13 @@ StoreWrapper::queryPathInfo(char * path, int base32) } SV * -StoreWrapper::queryRawRealisation(char * outputId) +StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) PPCODE: try { - auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput{ + .drvPath = THIS->store->parseStorePath(drvPath), + .outputName = outputName, + }); if (realisation) XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); else diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 2f6ebcef5..c4cbf2c96 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -49,14 +49,14 @@ fi clearStore nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore -rm -r "$REMOTE_STORE_DIR/realisations" +rm -r "$REMOTE_STORE_DIR/build-trace" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # There's no easy way to check whether a realisation is present on the local # store − short of manually querying the db, but the build environment doesn't # have the sqlite binary − so we instead push things again, and check that the # realisations have correctly been pushed to the remote store nix copy --to "$REMOTE_STORE" --file ../simple.nix -if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then +if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace")" ]]; then echo "Realisations not rebuilt" exit 1 fi @@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm "$REMOTE_STORE_DIR"/realisations/* +rm -r "$REMOTE_STORE_DIR"/build-trace/* buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 From d7464ecd9635497c314b7ea2ff4810a692cb1cc2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 00:24:12 -0400 Subject: [PATCH 09/13] Change JSON derivation format to use `ContentAddress` JSON Keeps it more consistent. --- .../source/protocols/json/derivation.md | 4 +-- doc/manual/source/protocols/json/meson.build | 2 +- ...{derivation-v3.yaml => derivation-v4.yaml} | 14 ++++++---- src/json-schema-checks/meson.build | 4 +-- .../ca/advanced-attributes-defaults.json | 2 +- ...-attributes-structured-attrs-defaults.json | 2 +- .../advanced-attributes-structured-attrs.json | 2 +- .../derivation/ca/advanced-attributes.json | 2 +- .../data/derivation/ca/self-contained.json | 2 +- .../data/derivation/dyn-dep-derivation.json | 2 +- .../ia/advanced-attributes-defaults.json | 2 +- ...-attributes-structured-attrs-defaults.json | 2 +- .../advanced-attributes-structured-attrs.json | 2 +- .../derivation/ia/advanced-attributes.json | 2 +- .../data/derivation/output-caFixedFlat.json | 7 +++-- .../data/derivation/output-caFixedNAR.json | 7 +++-- .../data/derivation/output-caFixedText.json | 7 +++-- .../data/derivation/simple-derivation.json | 2 +- src/libstore/derivations.cc | 27 ++++++++----------- tests/functional/dyn-drv/non-trivial.nix | 2 +- 20 files changed, 52 insertions(+), 44 deletions(-) rename doc/manual/source/protocols/json/schema/{derivation-v3.yaml => derivation-v4.yaml} (91%) diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 602ab67e4..d681771bd 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,7 +1,7 @@ -{{#include derivation-v3-fixed.md}} +{{#include derivation-v4-fixed.md}} diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 7f652ca0e..39b70d414 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -12,7 +12,7 @@ schemas = [ 'file-system-object-v1', 'hash-v1', 'content-address-v1', - 'derivation-v3', + 'derivation-v4', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml similarity index 91% rename from doc/manual/source/protocols/json/schema/derivation-v3.yaml rename to doc/manual/source/protocols/json/schema/derivation-v4.yaml index c5720ec6d..9380ee852 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -1,5 +1,5 @@ "$schema": http://json-schema.org/draft-04/schema# -"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v4.json title: Derivation description: | Experimental JSON representation of a Nix derivation (version 3). @@ -32,10 +32,10 @@ properties: Used when calculating store paths for the derivation’s outputs. version: - const: 3 + const: 4 title: Format version (must be 3) description: | - Must be `3`. + Must be `4`. This is a guard that allows us to continue evolving this format. The choice of `3` is fairly arbitrary, but corresponds to this informal version: @@ -47,6 +47,8 @@ properties: - Version 3: Drop store dir from store paths, just include base name. + - Version 4: Use canonical content address JSON format for floating content addressed derivation outputs. + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. outputs: @@ -161,9 +163,11 @@ properties: hashAlgo: title: Hash algorithm "$ref": "./hash-v1.yaml#/$defs/algorithm" + description: | + For an output which will be [content addressed], but the content address is not specified up front, the name of the hash algorithm used. When the content address is fixed, use `hash.hashAlgo` instead. hash: - type: string title: Expected hash value description: | - For fixed-output derivations, the expected content hash in base-16. + For fixed-output derivations, the expected content hash. + "$ref": "./hash-v1.yaml" diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 933098fe6..98a6649da 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -48,7 +48,7 @@ schemas = [ }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml', + 'schema' : schema_dir / 'derivation-v4.yaml', 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', @@ -57,7 +57,7 @@ schemas = [ # # Not sure how to make subschema work # { # 'stem': 'derivation', - # 'schema': schema_dir / 'derivation-v3.yaml#output', + # 'schema': schema_dir / 'derivation-v4.yaml#output', # 'files' : [ # 'output-caFixedFlat.json', # 'output-caFixedNAR.json', diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json index eb4bd4f3d..a391b7206 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -22,5 +22,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 3a4a3079b..1c0689539 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -33,5 +33,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index b10355af7..9d8f03fc9 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -101,5 +101,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index d66882036..e002b383f 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -52,5 +52,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/self-contained.json b/src/libstore-tests/data/derivation/ca/self-contained.json index 331beb7be..fed38b7ba 100644 --- a/src/libstore-tests/data/derivation/ca/self-contained.json +++ b/src/libstore-tests/data/derivation/ca/self-contained.json @@ -20,5 +20,5 @@ } }, "system": "x86_64-linux", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/dyn-dep-derivation.json b/src/libstore-tests/data/derivation/dyn-dep-derivation.json index 1a9f54c53..1d648e3d9 100644 --- a/src/libstore-tests/data/derivation/dyn-dep-derivation.json +++ b/src/libstore-tests/data/derivation/dyn-dep-derivation.json @@ -35,5 +35,5 @@ "name": "dyn-dep-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json index 0fa543f21..01b49dc3c 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json @@ -19,5 +19,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index e02392ea1..f2bb4af33 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -29,5 +29,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index 9230b06b6..b7acd78c8 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -96,5 +96,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index ba5911c91..37f86e4a2 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -49,5 +49,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json index e6a0123f6..9a38608b3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedFlat.json +++ b/src/libstore-tests/data/derivation/output-caFixedFlat.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "flat" } diff --git a/src/libstore-tests/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json index b57e065a9..767c605a3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedNAR.json +++ b/src/libstore-tests/data/derivation/output-caFixedNAR.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "nar" } diff --git a/src/libstore-tests/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json index 84778509e..a04f1ff2a 100644 --- a/src/libstore-tests/data/derivation/output-caFixedText.json +++ b/src/libstore-tests/data/derivation/output-caFixedText.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "text" } diff --git a/src/libstore-tests/data/derivation/simple-derivation.json b/src/libstore-tests/data/derivation/simple-derivation.json index 41a049aef..a3ad96a36 100644 --- a/src/libstore-tests/data/derivation/simple-derivation.json +++ b/src/libstore-tests/data/derivation/simple-derivation.json @@ -22,5 +22,5 @@ "name": "simple-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 20f1d6ca1..af083b1f1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1293,15 +1293,13 @@ void adl_serializer::to_json(json & res, const DerivationOutpu overloaded{ [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, [&](const DerivationOutput::CAFixed & dof) { - /* it would be nice to output the path for user convenience, but - this would require us to know the store dir. */ + res = dof.ca; + // FIXME print refs? + /* it would be nice to output the path for user convenience, but + this would require us to know the store dir. */ #if 0 res["path"] = dof.path(store, drvName, outputName); #endif - res["method"] = std::string{dof.ca.method.render()}; - res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); - res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); - // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { res["method"] = std::string{dof.method.render()}; @@ -1341,15 +1339,12 @@ adl_serializer::from_json(const json & _json, const Experiment }; } - else if (keys == (std::set{"method", "hashAlgo", "hash"})) { - auto [method, hashAlgo] = methodAlgo(); + else if (keys == (std::set{"method", "hash"})) { auto dof = DerivationOutput::CAFixed{ - .ca = - ContentAddress{ - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), - }, + .ca = static_cast(_json), }; + if (dof.ca.method == ContentAddressMethod::Raw::Text) + xpSettings.require(Xp::DynamicDerivations, "text-hashed derivation output in JSON"); /* We no longer produce this (denormalized) field (for the reasons described above), so we don't need to check it. */ #if 0 @@ -1392,7 +1387,7 @@ void adl_serializer::to_json(json & res, const Derivation & d) res["name"] = d.name; - res["version"] = 3; + res["version"] = 4; { nlohmann::json & outputsObj = res["outputs"]; @@ -1450,8 +1445,8 @@ Derivation adl_serializer::from_json(const json & _json, const Exper res.name = getString(valueAt(json, "name")); - if (valueAt(json, "version") != 3) - throw Error("Only derivation format version 3 is currently supported."); + if (valueAt(json, "version") != 4) + throw Error("Only derivation format version 4 is currently supported."); try { auto outputs = getObject(valueAt(json, "outputs")); diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix index 3c24ac2ee..e8f06d294 100644 --- a/tests/functional/dyn-drv/non-trivial.nix +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -63,7 +63,7 @@ builtins.outputOf } }, "system": "${system}", - "version": 3 + "version": 4 } EOF drvPath=$(echo "$json" | nix derivation add) From 436e3e779ac7102d976aadde665ae077367e7e6d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Sep 2025 08:43:21 -0400 Subject: [PATCH 10/13] Rework `ValidPathInfo`, `NarInfo` JSON instances Progress on #13570. If we depend on the store dir, our JSON serializers/deserializers take extra arguements, and that interfaces with the likes of various frameworks for associating these with types (e.g. nlohmann in C++, Serde in Rust, and Aeson in Haskell). --- src/libstore-tests/nar-info.cc | 36 ++++++++-------- src/libstore-tests/path-info.cc | 4 +- src/libstore/include/nix/store/nar-info.hh | 32 +++++++++++--- src/libstore/include/nix/store/path-info.hh | 18 +++++--- src/libstore/nar-info.cc | 39 ++++++++++++----- src/libstore/path-info.cc | 47 ++++++++++++++++----- src/nix/path-info.cc | 2 +- 7 files changed, 123 insertions(+), 55 deletions(-) diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 751c5e305..1add98053 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -59,24 +59,24 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) return info; } -#define JSON_TEST(STEM, PURE) \ - TEST_F(NarInfoTest, NarInfo_##STEM##_from_json) \ - { \ - readTest(#STEM, [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - auto expected = makeNarInfo(*store, PURE); \ - NarInfo got = NarInfo::fromJSON(*store, expected.path, encoded); \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(NarInfoTest, NarInfo_##STEM##_to_json) \ - { \ - writeTest( \ - #STEM, \ - [&]() -> json { return makeNarInfo(*store, PURE).toJSON(*store, PURE, HashFormat::SRI); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + auto got = UnkeyedNarInfo::fromJSON(&*store, encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return makeNarInfo(*store, PURE).toJSON(&*store, PURE); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } JSON_TEST(pure, false) diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 63310c1c3..8c02bf403 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -70,7 +70,7 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo { \ readTest(#STEM, [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ - UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(*store, encoded); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(&*store, encoded); \ auto expected = OBJ; \ ASSERT_EQ(got, expected); \ }); \ @@ -80,7 +80,7 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo { \ writeTest( \ #STEM, \ - [&]() -> json { return OBJ.toJSON(*store, PURE, HashFormat::SRI); }, \ + [&]() -> json { return OBJ.toJSON(&*store, PURE); }, \ [](const auto & file) { return json::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } diff --git a/src/libstore/include/nix/store/nar-info.hh b/src/libstore/include/nix/store/nar-info.hh index 1684837c6..92af0b7d5 100644 --- a/src/libstore/include/nix/store/nar-info.hh +++ b/src/libstore/include/nix/store/nar-info.hh @@ -9,17 +9,38 @@ namespace nix { struct StoreDirConfig; -struct NarInfo : ValidPathInfo +struct UnkeyedNarInfo : virtual UnkeyedValidPathInfo { std::string url; std::string compression; std::optional fileHash; uint64_t fileSize = 0; + UnkeyedNarInfo(UnkeyedValidPathInfo info) + : UnkeyedValidPathInfo(std::move(info)) + { + } + + bool operator==(const UnkeyedNarInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet + // auto operator <=>(const NarInfo &) const = default; + + nlohmann::json toJSON(const StoreDirConfig * store, bool includeImpureInfo) const override; + static UnkeyedNarInfo fromJSON(const StoreDirConfig * store, const nlohmann::json & json); +}; + +/** + * Key and the extra NAR fields + */ +struct NarInfo : ValidPathInfo, UnkeyedNarInfo +{ NarInfo() = delete; NarInfo(ValidPathInfo info) - : ValidPathInfo{std::move(info)} + : UnkeyedValidPathInfo(std::move(static_cast(info))) + // later moves will be partially ignored + , ValidPathInfo(std::move(info)) + , UnkeyedNarInfo(std::move(info)) { } @@ -37,13 +58,10 @@ struct NarInfo : ValidPathInfo NarInfo(const StoreDirConfig & store, const std::string & s, const std::string & whence); bool operator==(const NarInfo &) const = default; - // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet - // auto operator <=>(const NarInfo &) const = default; std::string to_string(const StoreDirConfig & store) const; - - nlohmann::json toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const override; - static NarInfo fromJSON(const StoreDirConfig & store, const StorePath & path, const nlohmann::json & json); }; } // namespace nix + +JSON_IMPL(nix::UnkeyedNarInfo) diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index cbc5abdb4..8f6115b73 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -117,11 +117,11 @@ struct UnkeyedValidPathInfo * @param includeImpureInfo If true, variable elements such as the * registration time are included. */ - virtual nlohmann::json toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const; - static UnkeyedValidPathInfo fromJSON(const StoreDirConfig & store, const nlohmann::json & json); + virtual nlohmann::json toJSON(const StoreDirConfig * store, bool includeImpureInfo) const; + static UnkeyedValidPathInfo fromJSON(const StoreDirConfig * store, const nlohmann::json & json); }; -struct ValidPathInfo : UnkeyedValidPathInfo +struct ValidPathInfo : virtual UnkeyedValidPathInfo { StorePath path; @@ -174,10 +174,14 @@ struct ValidPathInfo : UnkeyedValidPathInfo ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info) - , path(std::move(path)) {}; + , path(std::move(path)) + { + } + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) - : UnkeyedValidPathInfo(info) - , path(path) {}; + : ValidPathInfo(StorePath{path}, std::move(info)) + { + } static ValidPathInfo makeFromCA(const StoreDirConfig & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); @@ -191,3 +195,5 @@ static_assert(std::is_move_constructible_v); using ValidPathInfos = std::map; } // namespace nix + +JSON_IMPL(nix::UnkeyedValidPathInfo) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1e7c48287..7cf8d379c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,7 +7,9 @@ namespace nix { NarInfo::NarInfo(const StoreDirConfig & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack + : UnkeyedValidPathInfo(Hash::dummy) // FIXME: hack + , ValidPathInfo(StorePath::dummy, static_cast(*this)) // FIXME: hack + , UnkeyedNarInfo(static_cast(*this)) { unsigned line = 1; @@ -130,19 +132,23 @@ std::string NarInfo::to_string(const StoreDirConfig & store) const return res; } -nlohmann::json NarInfo::toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const +nlohmann::json UnkeyedNarInfo::toJSON(const StoreDirConfig * store, bool includeImpureInfo) const { using nlohmann::json; - auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + auto jsonObject = UnkeyedValidPathInfo::toJSON(store, includeImpureInfo); if (includeImpureInfo) { if (!url.empty()) jsonObject["url"] = url; if (!compression.empty()) jsonObject["compression"] = compression; - if (fileHash) - jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileHash) { + /* Back compat hack, see comment for "narHash" in + `UnkeyedValidPathInfo::toJSON` for details. */ + jsonObject["downloadHash"] = + store ? static_cast(fileHash->to_string(HashFormat::SRI, true)) : static_cast(*fileHash); + } if (fileSize) jsonObject["downloadSize"] = fileSize; } @@ -150,14 +156,11 @@ nlohmann::json NarInfo::toJSON(const StoreDirConfig & store, bool includeImpureI return jsonObject; } -NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const StorePath & path, const nlohmann::json & json) +UnkeyedNarInfo UnkeyedNarInfo::fromJSON(const StoreDirConfig * store, const nlohmann::json & json) { using nlohmann::detail::value_t; - NarInfo res{ValidPathInfo{ - path, - UnkeyedValidPathInfo::fromJSON(store, json), - }}; + UnkeyedNarInfo res{UnkeyedValidPathInfo::fromJSON(store, json)}; if (json.contains("url")) res.url = getString(valueAt(json, "url")); @@ -175,3 +178,19 @@ NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const StorePath & path, } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +UnkeyedNarInfo adl_serializer::from_json(const json & json) +{ + return UnkeyedNarInfo::fromJSON(nullptr, json); +} + +void adl_serializer::to_json(json & json, const UnkeyedNarInfo & c) +{ + json = c.toJSON(nullptr, true); +} + +} // namespace nlohmann diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 270c532bb..0ec7d1632 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -149,26 +149,32 @@ ValidPathInfo ValidPathInfo::makeFromCA( return res; } -nlohmann::json -UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const +nlohmann::json UnkeyedValidPathInfo::toJSON(const StoreDirConfig * store, bool includeImpureInfo) const { using nlohmann::json; auto jsonObject = json::object(); - jsonObject["narHash"] = narHash.to_string(hashFormat, true); + /* Back-compat hack, if we are passing a `StoreDirConfig`, do SRI, + which `nix path-info` has always down. Otherwise, use the new + cannonical JSON serialization for `Hash`. */ + jsonObject["narHash"] = + store ? static_cast(narHash.to_string(HashFormat::SRI, true)) : static_cast(narHash); jsonObject["narSize"] = narSize; { auto & jsonRefs = jsonObject["references"] = json::array(); for (auto & ref : references) - jsonRefs.emplace_back(store.printStorePath(ref)); + jsonRefs.emplace_back(store ? static_cast(store->printStorePath(ref)) : static_cast(ref)); } - jsonObject["ca"] = ca ? (std::optional{renderContentAddress(*ca)}) : std::nullopt; + jsonObject["ca"] = ca ? (store ? static_cast(renderContentAddress(*ca)) : static_cast(*ca)) + : static_cast(nullptr); if (includeImpureInfo) { - jsonObject["deriver"] = deriver ? (std::optional{store.printStorePath(*deriver)}) : std::nullopt; + jsonObject["deriver"] = deriver ? (store ? static_cast(std::optional{store->printStorePath(*deriver)}) + : static_cast(std::optional{*deriver})) + : static_cast(std::optional{}); jsonObject["registrationTime"] = registrationTime ? (std::optional{registrationTime}) : std::nullopt; @@ -182,20 +188,23 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf return jsonObject; } -UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store, const nlohmann::json & _json) +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig * store, const nlohmann::json & _json) { UnkeyedValidPathInfo res{ Hash(Hash::dummy), }; auto & json = getObject(_json); - res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); + res.narHash = [&] { + auto & j = valueAt(json, "narHash"); + return store ? Hash::parseAny(getString(j), std::nullopt) : static_cast(j); + }(); res.narSize = getUnsigned(valueAt(json, "narSize")); try { auto references = getStringList(valueAt(json, "references")); for (auto & input : references) - res.references.insert(store.parseStorePath(static_cast(input))); + res.references.insert(store ? store->parseStorePath(getString(input)) : static_cast(input)); } catch (Error & e) { e.addTrace({}, "while reading key 'references'"); throw; @@ -205,11 +214,11 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store // missing is for back-compat. if (json.contains("ca")) if (auto * rawCa = getNullable(valueAt(json, "ca"))) - res.ca = ContentAddress::parse(getString(*rawCa)); + res.ca = store ? ContentAddress::parse(getString(*rawCa)) : static_cast(*rawCa); if (json.contains("deriver")) if (auto * rawDeriver = getNullable(valueAt(json, "deriver"))) - res.deriver = store.parseStorePath(getString(*rawDeriver)); + res.deriver = store ? store->parseStorePath(getString(*rawDeriver)) : static_cast(*rawDeriver); if (json.contains("registrationTime")) if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime"))) @@ -225,3 +234,19 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +UnkeyedValidPathInfo adl_serializer::from_json(const json & json) +{ + return UnkeyedValidPathInfo::fromJSON(nullptr, json); +} + +void adl_serializer::to_json(json & json, const UnkeyedValidPathInfo & c) +{ + json = c.toJSON(nullptr, true); +} + +} // namespace nlohmann diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index fef3ae120..616e7989e 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -51,7 +51,7 @@ static json pathInfoToJSON(Store & store, const StorePathSet & storePaths, bool // know the name yet until we've read the NAR info. printedStorePath = store.printStorePath(info->path); - jsonObject = info->toJSON(store, true, HashFormat::SRI); + jsonObject = info->toJSON(&store, true); if (showClosureSize) { StorePathSet closure; From f446cb290c31d439e25e55bb03dd6f7483c52406 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Sep 2025 02:15:20 -0400 Subject: [PATCH 11/13] JSON impl for `DummyStore` --- .../data/dummy-store/empty.json | 5 ++ .../data/dummy-store/one-flat-file.json | 34 ++++++++ src/libstore-tests/dummy-store.cc | 83 ++++++++++++++++++ src/libstore/dummy-store.cc | 86 +++++++++++++++++++ .../include/nix/store/dummy-store-impl.hh | 10 +++ src/libstore/include/nix/store/dummy-store.hh | 7 ++ 6 files changed, 225 insertions(+) create mode 100644 src/libstore-tests/data/dummy-store/empty.json create mode 100644 src/libstore-tests/data/dummy-store/one-flat-file.json diff --git a/src/libstore-tests/data/dummy-store/empty.json b/src/libstore-tests/data/dummy-store/empty.json new file mode 100644 index 000000000..b22902334 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/empty.json @@ -0,0 +1,5 @@ +{ + "build-trace": {}, + "contents": {}, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json new file mode 100644 index 000000000..035accb78 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -0,0 +1,34 @@ +{ + "build-trace": {}, + "contents": { + "5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": { + "contents": { + "contents": "asdf", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "method": "nar" + }, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "narSize": 120, + "references": [], + "registrationTime": null, + "signatures": [], + "ultimate": false + } + } + }, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 3dd8137a3..2429bdc2b 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -1,11 +1,32 @@ #include +#include +#include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/globals.hh" #include "nix/store/realisation.hh" +#include "nix/util/tests/json-characterization.hh" + namespace nix { +class DummyStoreTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "dummy-store"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + static void SetUpTestSuite() + { + initLibStore(false); + } +}; + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); @@ -35,4 +56,66 @@ TEST(DummyStore, realisation_read) EXPECT_EQ(*value2, value); } +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +struct DummyStoreJsonTest : DummyStoreTest, + JsonCharacterizationTest>, + ::testing::WithParamInterface>> +{}; + +TEST_P(DummyStoreJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + using namespace nlohmann; + /* Cannot use `readJsonTest` because need to dereference the stores + for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + ref decoded = adl_serializer>::from_json(encoded); + ASSERT_EQ(*decoded, *expected); + }); +} + +TEST_P(DummyStoreJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { + initLibStore(false); + auto writeCfg = make_ref(DummyStore::Config::Params{}); + writeCfg->readOnly = false; + return ::testing::Values( + std::pair{ + "empty", + make_ref(DummyStore::Config::Params{})->openDummyStore(), + }, + std::pair{ + "one-flat-file", + [&] { + auto store = writeCfg->openDummyStore(); + store->addToStore( + "my-file", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "asdf", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + return store; + }(), + }); +}()); + } // namespace nix diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 5924f897c..0057783a9 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -2,6 +2,7 @@ #include "nix/util/archive.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/json-utils.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/realisation.hh" @@ -16,6 +17,16 @@ std::string DummyStoreConfig::doc() ; } +bool DummyStore::PathInfoAndContents::operator==(const PathInfoAndContents & other) const +{ + return info == other.info && contents->root == other.contents->root; +} + +bool DummyStore::operator==(const DummyStore & other) const +{ + return contents == other.contents && buildTrace == other.buildTrace; +} + namespace { class WholeStoreViewAccessor : public SourceAccessor @@ -378,3 +389,78 @@ ref DummyStore::Config::openDummyStore() const static RegisterStoreImplementation regDummyStore; } // namespace nix + +namespace nlohmann { + +using namespace nix; + +DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return DummyStore::PathInfoAndContents{ + .info = valueAt(obj, "info"), + .contents = make_ref(valueAt(obj, "contents")), + }; +} + +void adl_serializer::to_json(json & json, const DummyStore::PathInfoAndContents & val) +{ + json = { + {"info", val.info}, + {"contents", *val.contents}, + }; +} + +ref adl_serializer>::from_json(const json & json) +{ + auto & obj = getObject(json); + ref res = [&] { + auto cfg = make_ref(DummyStore::Config::Params{}); + const_cast(cfg->storeDir_).set(getString(valueAt(obj, "store-dir"))); + cfg->readOnly = true; + return cfg->openDummyStore(); + }(); + for (auto & [k, v] : getObject(valueAt(obj, "contents"))) + res->contents.insert({StorePath{k}, v}); + for (auto & [k0, v] : getObject(valueAt(obj, "build-trace"))) { + for (auto & [k1, v2] : getObject(v)) { + auto vref = make_ref(v2); + res->buildTrace.insert_or_visit( + { + Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + {{k1, vref}}, + }, + [&](auto & kv) { kv.second.insert_or_assign(k1, vref); }); + } + } + return res; +} + +void adl_serializer>::to_json(json & json, const ref & val) +{ + json = { + {"store-dir", val->storeDir}, + {"contents", + [&] { + auto obj = json::object(); + val->contents.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, + {"build-trace", + [&] { + auto obj = json::object(); + val->buildTrace.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + for (auto & [k2, v2] : kv.second) + obj2[k2] = *v2; + }); + return obj; + }()}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 137f81c9b..71869cb7e 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -23,6 +23,8 @@ struct DummyStore : virtual Store { UnkeyedValidPathInfo info; ref contents; + + bool operator==(const PathInfoAndContents &) const; }; /** @@ -54,6 +56,14 @@ struct DummyStore : virtual Store , config(config) { } + + bool operator==(const DummyStore &) const; }; +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::DummyStore::PathInfoAndContents) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index d371c4e51..6b7b20153 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/json-impls.hh" #include @@ -65,4 +66,10 @@ struct DummyStoreConfig : public std::enable_shared_from_this, } }; +template<> +struct json_avoids_null> : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::ref) From acd9dbfc556daef89ea1bf593027cda0d3ee7750 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 16:00:27 -0400 Subject: [PATCH 12/13] Update dummy store JSON format to include derivations --- src/libstore-tests/data/dummy-store/empty.json | 1 + .../data/dummy-store/one-derivation.json | 18 ++++++++++++++++++ .../data/dummy-store/one-flat-file.json | 1 + src/libstore-tests/dummy-store.cc | 10 ++++++++++ src/libstore/dummy-store.cc | 11 +++++++++++ 5 files changed, 41 insertions(+) create mode 100644 src/libstore-tests/data/dummy-store/one-derivation.json diff --git a/src/libstore-tests/data/dummy-store/empty.json b/src/libstore-tests/data/dummy-store/empty.json index b22902334..976a05843 100644 --- a/src/libstore-tests/data/dummy-store/empty.json +++ b/src/libstore-tests/data/dummy-store/empty.json @@ -1,5 +1,6 @@ { "build-trace": {}, "contents": {}, + "derivations": {}, "store-dir": "/nix/store" } diff --git a/src/libstore-tests/data/dummy-store/one-derivation.json b/src/libstore-tests/data/dummy-store/one-derivation.json new file mode 100644 index 000000000..201418db6 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-derivation.json @@ -0,0 +1,18 @@ +{ + "build-trace": {}, + "contents": {}, + "derivations": { + "rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": { + "args": [], + "builder": "", + "env": {}, + "inputDrvs": {}, + "inputSrcs": [], + "name": "foo", + "outputs": {}, + "system": "", + "version": 4 + } + }, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json index 035accb78..d98990baa 100644 --- a/src/libstore-tests/data/dummy-store/one-flat-file.json +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -30,5 +30,6 @@ } } }, + "derivations": {}, "store-dir": "/nix/store" } diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 2429bdc2b..36a0c3019 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -115,6 +115,16 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { HashAlgorithm::SHA256); return store; }(), + }, + std::pair{ + "one-derivation", + [&] { + auto store = writeCfg->openDummyStore(); + Derivation drv; + drv.name = "foo"; + store->writeDerivation(drv); + return store; + }(), }); }()); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 0057783a9..ba74ba826 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -422,6 +422,8 @@ ref adl_serializer>::from_json(const json & json) }(); for (auto & [k, v] : getObject(valueAt(obj, "contents"))) res->contents.insert({StorePath{k}, v}); + for (auto & [k, v] : getObject(valueAt(obj, "derivations"))) + res->derivations.insert({StorePath{k}, v}); for (auto & [k0, v] : getObject(valueAt(obj, "build-trace"))) { for (auto & [k1, v2] : getObject(v)) { auto vref = make_ref(v2); @@ -449,6 +451,15 @@ void adl_serializer>::to_json(json & json, const ref }); return obj; }()}, + {"derivations", + [&] { + auto obj = json::object(); + val->derivations.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, {"build-trace", [&] { auto obj = json::object(); From e59b4c69f1767056d0039a6679002226e021a7cb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Oct 2025 13:49:54 -0400 Subject: [PATCH 13/13] Fixup issues with bytes --- src/libstore-tests/dummy-store.cc | 3 ++- src/libstore/dummy-store.cc | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index f53fc03b4..3ac3c2651 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -1,6 +1,7 @@ #include #include +#include "nix/util/bytes.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/globals.hh" @@ -105,7 +106,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { auto sc = make_ref(); sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ .executable = false, - .contents = "asdf", + .contents = to_owned(as_bytes("asdf")), }}; return sc; }(), diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c811c1093..be063b1d5 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,5 +1,6 @@ #include "nix/store/store-registration.hh" #include "nix/util/archive.hh" +#include "nix/util/bytes.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/util/json-utils.hh" @@ -153,7 +154,7 @@ struct DummyStoreImpl : DummyStore /* compute path info on demand */ auto accessor = make_ref(); accessor->root = MemorySourceAccessor::File::Regular{ - .contents = kv.second.unparse(*this, false), + .contents = to_owned(as_bytes(kv.second.unparse(*this, false))), }; auto narHash = hashPath( {accessor, CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); @@ -164,7 +165,7 @@ struct DummyStoreImpl : DummyStore .method = ContentAddressMethod::Raw::Text, .hash = hashString( HashAlgorithm::SHA256, - std::get(accessor->root->raw).contents), + to_str(std::get(accessor->root->raw).contents)), }; callback(std::move(info)); })) @@ -429,7 +430,7 @@ ref adl_serializer>::from_json(const json & json) auto vref = make_ref(v2); res->buildTrace.insert_or_visit( { - Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + StorePath{k0}, {{k1, vref}}, }, [&](auto & kv) { kv.second.insert_or_assign(k1, vref); }); @@ -465,7 +466,7 @@ void adl_serializer>::to_json(json & json, const ref auto obj = json::object(); val->buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + auto & obj2 = obj[k.to_string()] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = *v2; });