From 389bcba97a1295440a24c887840b1af3e73f0dd3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 13:42:52 -0400 Subject: [PATCH 01/12] JSON Impl and schema for `BuildResult` --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/json/build-result.md | 21 +++ doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/build-result-v1 | 1 + .../json/schema/build-result-v1.yaml | 136 +++++++++++++++++ src/json-schema-checks/build-result | 1 + src/json-schema-checks/meson.build | 9 ++ src/json-schema-checks/package.nix | 1 + src/libstore-tests/build-result.cc | 108 +++++++++++++ .../data/build-result/not-deterministic.json | 9 ++ .../data/build-result/output-rejected.json | 9 ++ .../data/build-result/success.json | 23 +++ src/libstore-tests/meson.build | 1 + src/libstore/build-result.cc | 142 ++++++++++++++++++ .../include/nix/store/build-result.hh | 3 + 16 files changed, 467 insertions(+) create mode 100644 doc/manual/source/protocols/json/build-result.md create mode 120000 doc/manual/source/protocols/json/schema/build-result-v1 create mode 100644 doc/manual/source/protocols/json/schema/build-result-v1.yaml create mode 120000 src/json-schema-checks/build-result create mode 100644 src/libstore-tests/build-result.cc create mode 100644 src/libstore-tests/data/build-result/not-deterministic.json create mode 100644 src/libstore-tests/data/build-result/output-rejected.json create mode 100644 src/libstore-tests/data/build-result/success.json diff --git a/doc/manual/package.nix b/doc/manual/package.nix index e13c6f33d..343e40016 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -41,6 +41,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 580076ece..5be3d6a90 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -127,6 +127,7 @@ - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) + - [Build Result](protocols/json/build-result.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/build-result.md b/doc/manual/source/protocols/json/build-result.md new file mode 100644 index 000000000..527e7bcc0 --- /dev/null +++ b/doc/manual/source/protocols/json/build-result.md @@ -0,0 +1,21 @@ +{{#include build-result-v1-fixed.md}} + +## Examples + +### Successful build + +```json +{{#include schema/build-result-v1/success.json}} +``` + +### Failed build (output rejected) + +```json +{{#include schema/build-result-v1/output-rejected.json}} +``` + +### Failed build (non-deterministic) + +```json +{{#include schema/build-result-v1/not-deterministic.json}} +``` \ No newline at end of file diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index d8e94d68c..c56de49c7 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -16,6 +16,7 @@ schemas = [ 'derivation-v3', 'deriving-path-v1', 'build-trace-entry-v1', + 'build-result-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-result-v1 b/doc/manual/source/protocols/json/schema/build-result-v1 new file mode 120000 index 000000000..a143d2c50 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml new file mode 100644 index 000000000..31f59a44d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -0,0 +1,136 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-result-v1.json" +title: Build Result +description: | + This schema describes the JSON representation of Nix's `BuildResult` type, which represents the result of building a derivation or substituting store paths. + + Build results can represent either successful builds (with built outputs) or various types of failures. + +oneOf: + - "$ref": "#/$defs/success" + - "$ref": "#/$defs/failure" +type: object +required: + - success + - status +properties: + timesBuilt: + type: integer + minimum: 0 + title: Times built + description: | + How many times this build was performed. + + startTime: + type: integer + minimum: 0 + title: Start time + description: | + The start time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + stopTime: + type: integer + minimum: 0 + title: Stop time + description: | + The stop time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + cpuUser: + type: integer + minimum: 0 + title: User CPU time + description: | + User CPU time the build took, in microseconds. + + cpuSystem: + type: integer + minimum: 0 + title: System CPU time + description: | + System CPU time the build took, in microseconds. + +"$defs": + success: + type: object + title: Successful Build Result + description: | + Represents a successful build with built outputs. + required: + - success + - status + - builtOutputs + properties: + success: + const: true + title: Success indicator + description: | + Always true for successful build results. + + status: + type: string + title: Success status + description: | + Status string for successful builds. + enum: + - "Built" + - "Substituted" + - "AlreadyValid" + - "ResolvesToAlreadyValid" + + builtOutputs: + type: object + title: Built outputs + description: | + A mapping from output names to their build trace entries. + additionalProperties: + "$ref": "build-trace-entry-v1.yaml" + + failure: + type: object + title: Failed Build Result + description: | + Represents a failed build with error information. + required: + - success + - status + - errorMsg + properties: + success: + const: false + title: Success indicator + description: | + Always false for failed build results. + + status: + type: string + title: Failure status + description: | + Status string for failed builds. + enum: + - "PermanentFailure" + - "InputRejected" + - "OutputRejected" + - "TransientFailure" + - "CachedFailure" + - "TimedOut" + - "MiscFailure" + - "DependencyFailed" + - "LogLimitExceeded" + - "NotDeterministic" + - "NoSubstituters" + - "HashMismatch" + + errorMsg: + type: string + title: Error message + description: | + Information about the error if the build failed. + + isNonDeterministic: + type: boolean + title: Non-deterministic flag + description: | + If timesBuilt > 1, whether some builds did not produce the same result. + + Note that 'isNonDeterministic = false' does not mean the build is deterministic, + just that we don't have evidence of non-determinism. diff --git a/src/json-schema-checks/build-result b/src/json-schema-checks/build-result new file mode 120000 index 000000000..8010d0fdd --- /dev/null +++ b/src/json-schema-checks/build-result @@ -0,0 +1 @@ +../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index c2c7fbff4..65a2651b7 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -150,6 +150,15 @@ schemas += [ 'impure.json', ], }, + { + 'stem' : 'build-result', + 'schema' : schema_dir / 'build-result-v1.yaml', + 'files' : [ + 'success.json', + 'output-rejected.json', + 'not-deterministic.json', + ], + }, # Match exact variant { 'stem' : 'store-object-info', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 057a6e85b..5365fe75e 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -28,6 +28,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result ./. ]; diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc new file mode 100644 index 000000000..85e799c2a --- /dev/null +++ b/src/libstore-tests/build-result.cc @@ -0,0 +1,108 @@ +#include + +#include "nix/store/build-result.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class BuildResultTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "build-result"; + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct BuildResultJsonTest : BuildResultTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(BuildResultJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(BuildResultJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +using namespace std::literals::chrono_literals; + +INSTANTIATE_TEST_SUITE_P( + BuildResultJSON, + BuildResultJsonTest, + ::testing::Values( + std::pair{ + "not-deterministic", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = false, // Note: This field is separate from the status + }}, + .timesBuilt = 1, + }, + }, + std::pair{ + "output-rejected", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + .isNonDeterministic = false, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + }, + std::pair{ + "success", + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs{ + { + "foo", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + }, + }, + { + "bar", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + }, + }, + }, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + })); + +} // namespace nix diff --git a/src/libstore-tests/data/build-result/not-deterministic.json b/src/libstore-tests/data/build-result/not-deterministic.json new file mode 100644 index 000000000..c24a15795 --- /dev/null +++ b/src/libstore-tests/data/build-result/not-deterministic.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "NotDeterministic", + "stopTime": 0, + "success": false, + "timesBuilt": 1 +} diff --git a/src/libstore-tests/data/build-result/output-rejected.json b/src/libstore-tests/data/build-result/output-rejected.json new file mode 100644 index 000000000..9494bf4ec --- /dev/null +++ b/src/libstore-tests/data/build-result/output-rejected.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 30, + "status": "OutputRejected", + "stopTime": 50, + "success": false, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json new file mode 100644 index 000000000..4baadb547 --- /dev/null +++ b/src/libstore-tests/data/build-result/success.json @@ -0,0 +1,23 @@ +{ + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 4d464ad89..f76df8bcb 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -54,6 +54,7 @@ deps_private += gtest subdir('nix-meson-build-support/common') sources = files( + 'build-result.cc', 'common-protocol.cc', 'content-address.cc', 'derivation-advanced-attrs.cc', diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index ecbd27b49..e3d9e9085 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -1,4 +1,6 @@ #include "nix/store/build-result.hh" +#include "nix/util/json-utils.hh" +#include namespace nix { @@ -11,4 +13,144 @@ std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Succes bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default; std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default; +static constexpr std::array, 4> successStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Success::e, #e} + ENUM_ENTRY(Built), + ENUM_ENTRY(Substituted), + ENUM_ENTRY(AlreadyValid), + ENUM_ENTRY(ResolvesToAlreadyValid), +#undef ENUM_ENTRY +}}; + +static std::string_view successStatusToString(BuildResult::Success::Status status) +{ + for (const auto & [enumVal, str] : successStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown success status: %d", static_cast(status)); +} + +static BuildResult::Success::Status successStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : successStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result success status '%s'", str); +} + +static constexpr std::array, 12> failureStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Failure::e, #e} + ENUM_ENTRY(PermanentFailure), + ENUM_ENTRY(InputRejected), + ENUM_ENTRY(OutputRejected), + ENUM_ENTRY(TransientFailure), + ENUM_ENTRY(CachedFailure), + ENUM_ENTRY(TimedOut), + ENUM_ENTRY(MiscFailure), + ENUM_ENTRY(DependencyFailed), + ENUM_ENTRY(LogLimitExceeded), + ENUM_ENTRY(NotDeterministic), + ENUM_ENTRY(NoSubstituters), + ENUM_ENTRY(HashMismatch), +#undef ENUM_ENTRY +}}; + +static std::string_view failureStatusToString(BuildResult::Failure::Status status) +{ + for (const auto & [enumVal, str] : failureStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown failure status: %d", static_cast(status)); +} + +static BuildResult::Failure::Status failureStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : failureStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result failure status '%s'", str); +} + } // namespace nix + +namespace nlohmann { + +using namespace nix; + +void adl_serializer::to_json(json & res, const BuildResult & br) +{ + res = json::object(); + + // Common fields + res["timesBuilt"] = br.timesBuilt; + res["startTime"] = br.startTime; + res["stopTime"] = br.stopTime; + + if (br.cpuUser.has_value()) { + res["cpuUser"] = br.cpuUser->count(); + } + if (br.cpuSystem.has_value()) { + res["cpuSystem"] = br.cpuSystem->count(); + } + + // Handle success or failure variant + std::visit( + overloaded{ + [&](const BuildResult::Success & success) { + res["success"] = true; + res["status"] = successStatusToString(success.status); + res["builtOutputs"] = success.builtOutputs; + }, + [&](const BuildResult::Failure & failure) { + res["success"] = false; + res["status"] = failureStatusToString(failure.status); + res["errorMsg"] = failure.errorMsg; + res["isNonDeterministic"] = failure.isNonDeterministic; + }, + }, + br.inner); +} + +BuildResult adl_serializer::from_json(const json & _json) +{ + auto & json = getObject(_json); + + BuildResult br; + + // Common fields + br.timesBuilt = getUnsigned(valueAt(json, "timesBuilt")); + br.startTime = getUnsigned(valueAt(json, "startTime")); + br.stopTime = getUnsigned(valueAt(json, "stopTime")); + + if (auto cpuUser = optionalValueAt(json, "cpuUser")) { + br.cpuUser = std::chrono::microseconds(getUnsigned(*cpuUser)); + } + if (auto cpuSystem = optionalValueAt(json, "cpuSystem")) { + br.cpuSystem = std::chrono::microseconds(getUnsigned(*cpuSystem)); + } + + // Determine success or failure based on success field + bool success = getBoolean(valueAt(json, "success")); + std::string statusStr = getString(valueAt(json, "status")); + + if (success) { + BuildResult::Success s; + s.status = successStatusFromString(statusStr); + s.builtOutputs = valueAt(json, "builtOutputs"); + br.inner = std::move(s); + } else { + BuildResult::Failure f; + f.status = failureStatusFromString(statusStr); + f.errorMsg = getString(valueAt(json, "errorMsg")); + f.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")); + br.inner = std::move(f); + } + + return br; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index 0446c4038..4739232f8 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -7,6 +7,7 @@ #include "nix/store/derived-path.hh" #include "nix/store/realisation.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -175,3 +176,5 @@ struct KeyedBuildResult : BuildResult }; } // namespace nix + +JSON_IMPL(nix::BuildResult) From 9daef9cca23c8d7f1658c3941eae59f1735c391e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Nov 2025 02:21:24 -0500 Subject: [PATCH 02/12] Clean up `DerivationOptions` unit tests We now test equality on whole strucks much more often, which avoids forgetting to test for specific fields. --- .../derivation-advanced-attrs.cc | 534 +++++++++--------- 1 file changed, 252 insertions(+), 282 deletions(-) diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 02bc8fa24..c994b26d5 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -32,6 +32,33 @@ public: * to worry about race conditions if the tests run concurrently. */ ExperimentalFeatureSettings mockXpSettings; + + /** + * Helper function to test getRequiredSystemFeatures for a given derivation file + */ + void testRequiredSystemFeatures(const std::string & fileName, const StringSet & expectedFeatures) + { + this->readTest(fileName, [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); + EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures); + }); + } + + /** + * Helper function to test DerivationOptions parsing and comparison + */ + void testDerivationOptions( + const std::string & fileName, const DerivationOptions & expected, const StringSet & expectedSystemFeatures) + { + this->readTest(fileName, [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); + + EXPECT_EQ(options, expected); + EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures); + }); + } }; class CaDerivationAdvancedAttrsTest : public DerivationAdvancedAttrsTest @@ -100,33 +127,35 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph); +static const DerivationOptions advancedAttributes_defaults = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) { this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, ""); - EXPECT_EQ(options.noChroot, false); - EXPECT_EQ(options.impureHostDeps, StringSet{}); - EXPECT_EQ(options.impureEnvVars, StringSet{}); - EXPECT_EQ(options.allowLocalNetworking, false); - EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); - { - auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); - ASSERT_TRUE(checksForAllOutputs_ != nullptr); - auto & checksForAllOutputs = *checksForAllOutputs_; + EXPECT_EQ(options, advancedAttributes_defaults); - EXPECT_EQ(checksForAllOutputs.allowedReferences, std::nullopt); - EXPECT_EQ(checksForAllOutputs.allowedRequisites, std::nullopt); - EXPECT_EQ(checksForAllOutputs.disallowedReferences, StringSet{}); - EXPECT_EQ(checksForAllOutputs.disallowedRequisites, StringSet{}); - } EXPECT_EQ(options.canBuildLocally(*this->store, got), false); EXPECT_EQ(options.willBuildLocally(*this->store, got), false); EXPECT_EQ(options.substitutesAllowed(), true); @@ -136,152 +165,124 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults) { - this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); - }); + testRequiredSystemFeatures("advanced-attributes-defaults.drv", {}); }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults) { - this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); - }); + testRequiredSystemFeatures("advanced-attributes-defaults.drv", {"ca-derivations"}); }; TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) { + DerivationOptions expected = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, + }; + this->readTest("advanced-attributes.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); - EXPECT_EQ(options.noChroot, true); - EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"}); - EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); - EXPECT_EQ(options.allowLocalNetworking, true); - EXPECT_EQ(options.canBuildLocally(*this->store, got), false); - EXPECT_EQ(options.willBuildLocally(*this->store, got), false); + // Reset fields that vary between test cases to enable whole-object comparison + options.outputChecks = DerivationOptions::OutputChecks{.ignoreSelfRefs = true}; + options.exportReferencesGraph = {}; + + EXPECT_EQ(options, expected); + EXPECT_EQ(options.substitutesAllowed(), false); EXPECT_EQ(options.useUidRange(got), true); }); }; -TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) +DerivationOptions advancedAttributes_ia = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + .disallowedReferences = StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + .allowedRequisites = StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, + .disallowedRequisites = StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}}, + {"refs2", {"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_ia) { - this->readTest("advanced-attributes.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"}); +}; - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", - }, - }, - { - "refs2", - { - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", - }, - }, - })); - - { - auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); - ASSERT_TRUE(checksForAllOutputs_ != nullptr); - auto & checksForAllOutputs = *checksForAllOutputs_; - - EXPECT_EQ( - checksForAllOutputs.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); - EXPECT_EQ( - checksForAllOutputs.allowedRequisites, - StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); - EXPECT_EQ( - checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); - EXPECT_EQ( - checksForAllOutputs.disallowedRequisites, - StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); +DerivationOptions advancedAttributes_ca = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}, + .disallowedReferences = StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}, + .allowedRequisites = StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}, + .disallowedRequisites = StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}}, + {"refs2", {"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) { - this->readTest("advanced-attributes.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"}); +}; - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", - }, - }, - { - "refs2", - { - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", - }, - }, - })); - - { - auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); - ASSERT_TRUE(checksForAllOutputs_ != nullptr); - auto & checksForAllOutputs = *checksForAllOutputs_; - - EXPECT_EQ( - checksForAllOutputs.allowedReferences, - StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); - EXPECT_EQ( - checksForAllOutputs.allowedRequisites, - StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); - EXPECT_EQ( - checksForAllOutputs.disallowedReferences, - StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); - EXPECT_EQ( - checksForAllOutputs.disallowedRequisites, - StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - systemFeatures.insert("ca-derivations"); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); +DerivationOptions advancedAttributes_structuredAttrs_defaults = { + .outputChecks = std::map{}, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, }; TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_defaults) @@ -289,26 +290,11 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, ""); - EXPECT_EQ(options.noChroot, false); - EXPECT_EQ(options.impureHostDeps, StringSet{}); - EXPECT_EQ(options.impureEnvVars, StringSet{}); - EXPECT_EQ(options.allowLocalNetworking, false); - EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); - - { - auto * checksPerOutput_ = std::get_if<1>(&options.outputChecks); - ASSERT_TRUE(checksPerOutput_ != nullptr); - auto & checksPerOutput = *checksPerOutput_; - - EXPECT_EQ(checksPerOutput.size(), 0u); - } + EXPECT_EQ(options, advancedAttributes_structuredAttrs_defaults); EXPECT_EQ(options.canBuildLocally(*this->store, got), false); EXPECT_EQ(options.willBuildLocally(*this->store, got), false); @@ -319,55 +305,61 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) { - this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); - }); + testRequiredSystemFeatures("advanced-attributes-structured-attrs-defaults.drv", {}); }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) { - this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); - }); + testRequiredSystemFeatures("advanced-attributes-structured-attrs-defaults.drv", {"ca-derivations"}); }; TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { + DerivationOptions expected = { + .outputChecks = + std::map{ + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, + }; + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); - EXPECT_EQ(options.noChroot, true); - EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"}); - EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); - EXPECT_EQ(options.allowLocalNetworking, true); - + // Reset fields that vary between test cases to enable whole-object comparison { - auto output_ = get(std::get<1>(options.outputChecks), "dev"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.maxSize, 789); - EXPECT_EQ(output.maxClosureSize, 5909); + // Delete all keys but "dev" in options.outputChecks + auto * outputChecksMapP = + std::get_if>(&options.outputChecks); + ASSERT_TRUE(outputChecksMapP); + auto & outputChecksMap = *outputChecksMapP; + auto devEntry = outputChecksMap.find("dev"); + ASSERT_TRUE(devEntry != outputChecksMap.end()); + auto devChecks = devEntry->second; + outputChecksMap.clear(); + outputChecksMap.emplace("dev", std::move(devChecks)); } + options.exportReferencesGraph = {}; + + EXPECT_EQ(options, expected); EXPECT_EQ(options.canBuildLocally(*this->store, got), false); EXPECT_EQ(options.willBuildLocally(*this->store, got), false); @@ -376,112 +368,90 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) }); }; +DerivationOptions advancedAttributes_structuredAttrs_ia = { + .outputChecks = + std::map{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + .allowedRequisites = StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + .disallowedRequisites = StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = + { + {"refs1", {"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}}, + {"refs2", {"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) { - this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", - }, - }, - { - "refs2", - { - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", - }, - }, - })); + testDerivationOptions( + "advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"}); +}; +DerivationOptions advancedAttributes_structuredAttrs_ca = { + .outputChecks = + std::map{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}, + .allowedRequisites = StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}, + .disallowedRequisites = StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = { - { - auto output_ = get(std::get<1>(options.outputChecks), "out"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); - EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); - } - - { - auto output_ = get(std::get<1>(options.outputChecks), "bin"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); - EXPECT_EQ( - output.disallowedRequisites, StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); - } - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); + {"refs1", {"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}}, + {"refs2", {"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) { - this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", - }, - }, - { - "refs2", - { - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", - }, - }, - })); - - { - { - auto output_ = get(std::get<1>(options.outputChecks), "out"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.allowedReferences, StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); - EXPECT_EQ(output.allowedRequisites, StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); - } - - { - auto output_ = get(std::get<1>(options.outputChecks), "bin"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ( - output.disallowedReferences, StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); - EXPECT_EQ( - output.disallowedRequisites, StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); - } - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - systemFeatures.insert("ca-derivations"); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); + testDerivationOptions( + "advanced-attributes-structured-attrs.drv", + advancedAttributes_structuredAttrs_ca, + {"rainbow", "uid-range", "ca-derivations"}); }; } // namespace nix From d05e85e5befbc7fcc19a28d459ed17f5e98c6f23 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Nov 2025 01:22:20 -0500 Subject: [PATCH 03/12] Fix `DerivationOptions` JSON implementation and test --- .../data/derivation/ca/all_set.json | 46 +++++++++++++ .../ca/structuredAttrs_all_set.json | 66 +++++++++++++++++++ .../data/derivation/ia/all_set.json | 46 +++++++++++++ .../data/derivation/ia/defaults.json | 24 +++++++ .../ia/structuredAttrs_all_set.json | 66 +++++++++++++++++++ .../ia/structuredAttrs_defaults.json | 16 +++++ .../derivation-advanced-attrs.cc | 25 ++++++- src/libstore/derivation-options.cc | 33 ++++++++-- src/libstore/derivations.cc | 2 +- src/libutil/include/nix/util/json-utils.hh | 11 ++++ src/libutil/json-utils.cc | 9 +-- 11 files changed, 326 insertions(+), 18 deletions(-) create mode 100644 src/libstore-tests/data/derivation/ca/all_set.json create mode 100644 src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json create mode 100644 src/libstore-tests/data/derivation/ia/all_set.json create mode 100644 src/libstore-tests/data/derivation/ia/defaults.json create mode 100644 src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json create mode 100644 src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json diff --git a/src/libstore-tests/data/derivation/ca/all_set.json b/src/libstore-tests/data/derivation/ca/all_set.json new file mode 100644 index 000000000..e06eada01 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/all_set.json @@ -0,0 +1,46 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "refs2": [ + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "allowedRequisites": [ + "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z" + ], + "disallowedReferences": [ + "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g" + ], + "disallowedRequisites": [ + "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8" + ], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json b/src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json new file mode 100644 index 000000000..2a321897c --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json @@ -0,0 +1,66 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "refs2": [ + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "perOutput": { + "bin": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [ + "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g" + ], + "disallowedRequisites": [ + "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8" + ], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + }, + "dev": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": 5909, + "maxSize": 789 + }, + "out": { + "allowedReferences": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "allowedRequisites": [ + "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z" + ], + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + } + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/all_set.json b/src/libstore-tests/data/derivation/ia/all_set.json new file mode 100644 index 000000000..62b6cdf97 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/all_set.json @@ -0,0 +1,46 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "refs2": [ + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "allowedRequisites": [ + "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev" + ], + "disallowedReferences": [ + "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar" + ], + "disallowedRequisites": [ + "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" + ], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/defaults.json b/src/libstore-tests/data/derivation/ia/defaults.json new file mode 100644 index 000000000..79a68989c --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/defaults.json @@ -0,0 +1,24 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": {}, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json b/src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json new file mode 100644 index 000000000..0fa383589 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json @@ -0,0 +1,66 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "refs2": [ + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "perOutput": { + "bin": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [ + "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar" + ], + "disallowedRequisites": [ + "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" + ], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + }, + "dev": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": 5909, + "maxSize": 789 + }, + "out": { + "allowedReferences": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "allowedRequisites": [ + "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev" + ], + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + } + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json b/src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json new file mode 100644 index 000000000..d898d85f5 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json @@ -0,0 +1,16 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": {}, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index c994b26d5..41538cdcc 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -10,13 +10,15 @@ #include "nix/util/json-utils.hh" #include "nix/store/tests/libstore.hh" -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { using namespace nlohmann; -class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest +class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, + public JsonCharacterizationTest, + public LibStoreTest { protected: std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "ia"; @@ -454,4 +456,23 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) {"rainbow", "uid-range", "ca-derivations"}); }; +#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ + TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ + { \ + this->JsonCharacterizationTest::readJsonTest(#VAR, advancedAttributes_##VAR2); \ + } \ + TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ + { \ + this->JsonCharacterizationTest::writeJsonTest(#VAR, advancedAttributes_##VAR2); \ + } + +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, all_set, ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, all_set, ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_defaults, structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ca) + +#undef TEST_JSON_OPTIONS + } // namespace nix diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index bd9704b44..75313841c 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -176,13 +176,26 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt return {}; }; - checks.allowedReferences = get_("allowedReferences"); - checks.allowedRequisites = get_("allowedRequisites"); - checks.disallowedReferences = get_("disallowedReferences").value_or(StringSet{}); - checks.disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}); - ; - - res.insert_or_assign(outputName, std::move(checks)); + res.insert_or_assign( + outputName, + OutputChecks{ + .maxSize = [&]() -> std::optional { + if (auto maxSize = get(output, "maxSize")) + return maxSize->get(); + else + return std::nullopt; + }(), + .maxClosureSize = [&]() -> std::optional { + if (auto maxClosureSize = get(output, "maxClosureSize")) + return maxClosureSize->get(); + else + return std::nullopt; + }(), + .allowedReferences = get_("allowedReferences"), + .disallowedReferences = get_("disallowedReferences").value_or(StringSet{}), + .allowedRequisites = get_("allowedRequisites"), + .disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}), + }); } } return res; @@ -364,6 +377,7 @@ DerivationOptions adl_serializer::from_json(const json & json .unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"), .passAsFile = getStringSet(valueAt(json, "passAsFile")), + .exportReferencesGraph = getMap(getObject(valueAt(json, "exportReferencesGraph")), getStringSet), .additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")), .noChroot = getBoolean(valueAt(json, "noChroot")), @@ -396,6 +410,7 @@ void adl_serializer::to_json(json & json, const DerivationOpt json["unsafeDiscardReferences"] = o.unsafeDiscardReferences; json["passAsFile"] = o.passAsFile; + json["exportReferencesGraph"] = o.exportReferencesGraph; json["additionalSandboxProfile"] = o.additionalSandboxProfile; json["noChroot"] = o.noChroot; @@ -423,6 +438,8 @@ DerivationOptions::OutputChecks adl_serializer: return { .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), + .maxSize = ptrToOwned(getNullable(valueAt(json, "maxSize"))), + .maxClosureSize = ptrToOwned(getNullable(valueAt(json, "maxClosureSize"))), .allowedReferences = ptrToOwned(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")), .allowedRequisites = ptrToOwned(getNullable(valueAt(json, "allowedRequisites"))), @@ -433,6 +450,8 @@ DerivationOptions::OutputChecks adl_serializer: void adl_serializer::to_json(json & json, const DerivationOptions::OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; + json["maxSize"] = c.maxSize; + json["maxClosureSize"] = c.maxClosureSize; json["allowedReferences"] = c.allowedReferences; json["disallowedReferences"] = c.disallowedReferences; json["allowedRequisites"] = c.allowedRequisites; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 726143db2..e6ac08fd9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1404,7 +1404,7 @@ void adl_serializer::to_json(json & res, const Derivation & d) { auto & inputsList = res["inputSrcs"]; - inputsList = nlohmann::json ::array(); + inputsList = nlohmann::json::array(); for (auto & input : d.inputSrcs) inputsList.emplace_back(input); } diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index 51ebb2b6c..7a3fe4f36 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -59,6 +59,17 @@ auto getInteger(const nlohmann::json & value) -> std::enable_if_t +std::map getMap(const nlohmann::json::object_t & jsonObject, auto && f) +{ + std::map map; + + for (const auto & [key, value] : jsonObject) + map.insert_or_assign(key, f(value)); + + return map; +} + const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value); Strings getStringList(const nlohmann::json & value); StringMap getStringMap(const nlohmann::json & value); diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 1502384e9..80779541e 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -91,14 +91,7 @@ Strings getStringList(const nlohmann::json & value) StringMap getStringMap(const nlohmann::json & value) { - auto & jsonObject = getObject(value); - - StringMap stringMap; - - for (const auto & [key, value] : jsonObject) - stringMap[getString(key)] = getString(value); - - return stringMap; + return getMap>(getObject(value), getString); } StringSet getStringSet(const nlohmann::json & value) From e7b274f85a1f75e3c4dba445f081dd7862586d07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Nov 2025 23:38:45 -0500 Subject: [PATCH 04/12] `DerivationBuilderParams` have reference to `BasicDerivation` Have one to that instead of one to `Derivation`. `DerivationBuilder` doesn't need `inputDrvs`, so `BasicDerivation` suffices. (In fact, it doesn't need `inputSrcs` either, but we don't yet hve a type to exclude that.) --- src/libstore/include/nix/store/build/derivation-builder.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index 5fad26e83..5eed38462 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -62,7 +62,7 @@ struct DerivationBuilderParams /** * The derivation stored at drvPath. */ - const Derivation & drv; + const BasicDerivation & drv; /** * The derivation options of `drv`. From c1317017e902f34631f2f4598710114d36e84ee4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Nov 2025 13:06:08 +0100 Subject: [PATCH 05/12] Don't crash on flakerefs containing newlines Fixes #14311. --- src/libflake/flakeref.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index a26f269c3..ed6b657ac 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -109,7 +109,8 @@ std::pair parsePathFlakeRefWithFragment( std::smatch match; auto succeeds = std::regex_match(url, match, pathFlakeRegex); - assert(succeeds); + if (!succeeds) + throw Error("invalid flakeref '%s'", url); auto path = match[1].str(); auto query = decodeQuery(match[3].str(), /*lenient=*/true); auto fragment = percentDecode(match[5].str()); From 8dfab7c2ddbca007f3f671835248e98c5a5c764c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Sep 2025 08:43:21 -0400 Subject: [PATCH 06/12] Rework `ValidPathInfo`, `NarInfo` JSON instances Progress on #13570. If we depend on the store dir, our JSON serializers/deserializers take extra arguements, and that interfaces with the likes of various frameworks for associating these with types (e.g. nlohmann in C++, Serde in Rust, and Aeson in Haskell). --- src/libstore-tests/nar-info.cc | 36 ++++++++-------- src/libstore-tests/path-info.cc | 4 +- src/libstore/include/nix/store/nar-info.hh | 32 +++++++++++--- src/libstore/include/nix/store/path-info.hh | 18 +++++--- src/libstore/nar-info.cc | 39 ++++++++++++----- src/libstore/path-info.cc | 47 ++++++++++++++++----- src/nix/path-info.cc | 2 +- 7 files changed, 123 insertions(+), 55 deletions(-) diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 751c5e305..1add98053 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -59,24 +59,24 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) return info; } -#define JSON_TEST(STEM, PURE) \ - TEST_F(NarInfoTest, NarInfo_##STEM##_from_json) \ - { \ - readTest(#STEM, [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - auto expected = makeNarInfo(*store, PURE); \ - NarInfo got = NarInfo::fromJSON(*store, expected.path, encoded); \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(NarInfoTest, NarInfo_##STEM##_to_json) \ - { \ - writeTest( \ - #STEM, \ - [&]() -> json { return makeNarInfo(*store, PURE).toJSON(*store, PURE, HashFormat::SRI); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + auto got = UnkeyedNarInfo::fromJSON(&*store, encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return makeNarInfo(*store, PURE).toJSON(&*store, PURE); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } JSON_TEST(pure, false) diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 63310c1c3..8c02bf403 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -70,7 +70,7 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo { \ readTest(#STEM, [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ - UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(*store, encoded); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(&*store, encoded); \ auto expected = OBJ; \ ASSERT_EQ(got, expected); \ }); \ @@ -80,7 +80,7 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo { \ writeTest( \ #STEM, \ - [&]() -> json { return OBJ.toJSON(*store, PURE, HashFormat::SRI); }, \ + [&]() -> json { return OBJ.toJSON(&*store, PURE); }, \ [](const auto & file) { return json::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } diff --git a/src/libstore/include/nix/store/nar-info.hh b/src/libstore/include/nix/store/nar-info.hh index 1684837c6..92af0b7d5 100644 --- a/src/libstore/include/nix/store/nar-info.hh +++ b/src/libstore/include/nix/store/nar-info.hh @@ -9,17 +9,38 @@ namespace nix { struct StoreDirConfig; -struct NarInfo : ValidPathInfo +struct UnkeyedNarInfo : virtual UnkeyedValidPathInfo { std::string url; std::string compression; std::optional fileHash; uint64_t fileSize = 0; + UnkeyedNarInfo(UnkeyedValidPathInfo info) + : UnkeyedValidPathInfo(std::move(info)) + { + } + + bool operator==(const UnkeyedNarInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet + // auto operator <=>(const NarInfo &) const = default; + + nlohmann::json toJSON(const StoreDirConfig * store, bool includeImpureInfo) const override; + static UnkeyedNarInfo fromJSON(const StoreDirConfig * store, const nlohmann::json & json); +}; + +/** + * Key and the extra NAR fields + */ +struct NarInfo : ValidPathInfo, UnkeyedNarInfo +{ NarInfo() = delete; NarInfo(ValidPathInfo info) - : ValidPathInfo{std::move(info)} + : UnkeyedValidPathInfo(std::move(static_cast(info))) + // later moves will be partially ignored + , ValidPathInfo(std::move(info)) + , UnkeyedNarInfo(std::move(info)) { } @@ -37,13 +58,10 @@ struct NarInfo : ValidPathInfo NarInfo(const StoreDirConfig & store, const std::string & s, const std::string & whence); bool operator==(const NarInfo &) const = default; - // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet - // auto operator <=>(const NarInfo &) const = default; std::string to_string(const StoreDirConfig & store) const; - - nlohmann::json toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const override; - static NarInfo fromJSON(const StoreDirConfig & store, const StorePath & path, const nlohmann::json & json); }; } // namespace nix + +JSON_IMPL(nix::UnkeyedNarInfo) diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index cbc5abdb4..8f6115b73 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -117,11 +117,11 @@ struct UnkeyedValidPathInfo * @param includeImpureInfo If true, variable elements such as the * registration time are included. */ - virtual nlohmann::json toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const; - static UnkeyedValidPathInfo fromJSON(const StoreDirConfig & store, const nlohmann::json & json); + virtual nlohmann::json toJSON(const StoreDirConfig * store, bool includeImpureInfo) const; + static UnkeyedValidPathInfo fromJSON(const StoreDirConfig * store, const nlohmann::json & json); }; -struct ValidPathInfo : UnkeyedValidPathInfo +struct ValidPathInfo : virtual UnkeyedValidPathInfo { StorePath path; @@ -174,10 +174,14 @@ struct ValidPathInfo : UnkeyedValidPathInfo ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info) - , path(std::move(path)) {}; + , path(std::move(path)) + { + } + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) - : UnkeyedValidPathInfo(info) - , path(path) {}; + : ValidPathInfo(StorePath{path}, std::move(info)) + { + } static ValidPathInfo makeFromCA(const StoreDirConfig & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); @@ -191,3 +195,5 @@ static_assert(std::is_move_constructible_v); using ValidPathInfos = std::map; } // namespace nix + +JSON_IMPL(nix::UnkeyedValidPathInfo) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 6f1abb273..55f455298 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,7 +7,9 @@ namespace nix { NarInfo::NarInfo(const StoreDirConfig & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack + : UnkeyedValidPathInfo(Hash::dummy) // FIXME: hack + , ValidPathInfo(StorePath::dummy, static_cast(*this)) // FIXME: hack + , UnkeyedNarInfo(static_cast(*this)) { unsigned line = 1; @@ -130,19 +132,23 @@ std::string NarInfo::to_string(const StoreDirConfig & store) const return res; } -nlohmann::json NarInfo::toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const +nlohmann::json UnkeyedNarInfo::toJSON(const StoreDirConfig * store, bool includeImpureInfo) const { using nlohmann::json; - auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + auto jsonObject = UnkeyedValidPathInfo::toJSON(store, includeImpureInfo); if (includeImpureInfo) { if (!url.empty()) jsonObject["url"] = url; if (!compression.empty()) jsonObject["compression"] = compression; - if (fileHash) - jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileHash) { + /* Back compat hack, see comment for "narHash" in + `UnkeyedValidPathInfo::toJSON` for details. */ + jsonObject["downloadHash"] = + store ? static_cast(fileHash->to_string(HashFormat::SRI, true)) : static_cast(*fileHash); + } if (fileSize) jsonObject["downloadSize"] = fileSize; } @@ -150,14 +156,11 @@ nlohmann::json NarInfo::toJSON(const StoreDirConfig & store, bool includeImpureI return jsonObject; } -NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const StorePath & path, const nlohmann::json & json) +UnkeyedNarInfo UnkeyedNarInfo::fromJSON(const StoreDirConfig * store, const nlohmann::json & json) { using nlohmann::detail::value_t; - NarInfo res{ValidPathInfo{ - path, - UnkeyedValidPathInfo::fromJSON(store, json), - }}; + UnkeyedNarInfo res{UnkeyedValidPathInfo::fromJSON(store, json)}; auto & obj = getObject(json); @@ -177,3 +180,19 @@ NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const StorePath & path, } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +UnkeyedNarInfo adl_serializer::from_json(const json & json) +{ + return UnkeyedNarInfo::fromJSON(nullptr, json); +} + +void adl_serializer::to_json(json & json, const UnkeyedNarInfo & c) +{ + json = c.toJSON(nullptr, true); +} + +} // namespace nlohmann diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 09a78a4ad..79c1dc8ac 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -149,26 +149,32 @@ ValidPathInfo ValidPathInfo::makeFromCA( return res; } -nlohmann::json -UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInfo, HashFormat hashFormat) const +nlohmann::json UnkeyedValidPathInfo::toJSON(const StoreDirConfig * store, bool includeImpureInfo) const { using nlohmann::json; auto jsonObject = json::object(); - jsonObject["narHash"] = narHash.to_string(hashFormat, true); + /* Back-compat hack, if we are passing a `StoreDirConfig`, do SRI, + which `nix path-info` has always down. Otherwise, use the new + cannonical JSON serialization for `Hash`. */ + jsonObject["narHash"] = + store ? static_cast(narHash.to_string(HashFormat::SRI, true)) : static_cast(narHash); jsonObject["narSize"] = narSize; { auto & jsonRefs = jsonObject["references"] = json::array(); for (auto & ref : references) - jsonRefs.emplace_back(store.printStorePath(ref)); + jsonRefs.emplace_back(store ? static_cast(store->printStorePath(ref)) : static_cast(ref)); } - jsonObject["ca"] = ca ? (std::optional{renderContentAddress(*ca)}) : std::nullopt; + jsonObject["ca"] = ca ? (store ? static_cast(renderContentAddress(*ca)) : static_cast(*ca)) + : static_cast(nullptr); if (includeImpureInfo) { - jsonObject["deriver"] = deriver ? (std::optional{store.printStorePath(*deriver)}) : std::nullopt; + jsonObject["deriver"] = deriver ? (store ? static_cast(std::optional{store->printStorePath(*deriver)}) + : static_cast(std::optional{*deriver})) + : static_cast(std::optional{}); jsonObject["registrationTime"] = registrationTime ? (std::optional{registrationTime}) : std::nullopt; @@ -182,20 +188,23 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf return jsonObject; } -UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store, const nlohmann::json & _json) +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig * store, const nlohmann::json & _json) { UnkeyedValidPathInfo res{ Hash(Hash::dummy), }; auto & json = getObject(_json); - res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); + res.narHash = [&] { + auto & j = valueAt(json, "narHash"); + return store ? Hash::parseAny(getString(j), std::nullopt) : static_cast(j); + }(); res.narSize = getUnsigned(valueAt(json, "narSize")); try { auto references = getStringList(valueAt(json, "references")); for (auto & input : references) - res.references.insert(store.parseStorePath(static_cast(input))); + res.references.insert(store ? store->parseStorePath(getString(input)) : static_cast(input)); } catch (Error & e) { e.addTrace({}, "while reading key 'references'"); throw; @@ -205,11 +214,11 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store // missing is for back-compat. if (auto * rawCa0 = optionalValueAt(json, "ca")) if (auto * rawCa = getNullable(*rawCa0)) - res.ca = ContentAddress::parse(getString(*rawCa)); + res.ca = store ? ContentAddress::parse(getString(*rawCa)) : static_cast(*rawCa); if (auto * rawDeriver0 = optionalValueAt(json, "deriver")) if (auto * rawDeriver = getNullable(*rawDeriver0)) - res.deriver = store.parseStorePath(getString(*rawDeriver)); + res.deriver = store ? store->parseStorePath(getString(*rawDeriver)) : static_cast(*rawDeriver); if (auto * rawRegistrationTime0 = optionalValueAt(json, "registrationTime")) if (auto * rawRegistrationTime = getNullable(*rawRegistrationTime0)) @@ -225,3 +234,19 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +UnkeyedValidPathInfo adl_serializer::from_json(const json & json) +{ + return UnkeyedValidPathInfo::fromJSON(nullptr, json); +} + +void adl_serializer::to_json(json & json, const UnkeyedValidPathInfo & c) +{ + json = c.toJSON(nullptr, true); +} + +} // namespace nlohmann diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 146b775e5..74921126b 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -51,7 +51,7 @@ static json pathInfoToJSON(Store & store, const StorePathSet & storePaths, bool // know the name yet until we've read the NAR info. printedStorePath = store.printStorePath(info->path); - jsonObject = info->toJSON(store, true, HashFormat::SRI); + jsonObject = info->toJSON(&store, true); if (showClosureSize) { StorePathSet closure; From 8026dd1ba4f2269e3b5afef1d399dccae6edebd1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 14:47:05 -0400 Subject: [PATCH 07/12] JSON alongside proto serialization test data This makes the proto serializer characterisation test data be accompanied by JSON data. This is useful because the JSON data is human-readable while the binary data is not. --- .../include/nix/store/tests/protocol.hh | 52 +++++++++++---- src/libstore-tests/common-protocol.cc | 13 +++- .../data/common-protocol/content-address.json | 26 ++++++++ .../data/common-protocol/drv-output.json | 4 ++ .../optional-content-address.json | 11 ++++ .../common-protocol/optional-store-path.json | 4 ++ .../data/common-protocol/realisation.json | 22 +++++++ .../data/common-protocol/set.json | 22 +++++++ .../data/common-protocol/store-path.json | 4 ++ .../data/common-protocol/string.json | 7 +++ .../data/common-protocol/vector.json | 22 +++++++ .../data/serve-protocol/build-result-2.2.json | 28 +++++++++ .../data/serve-protocol/build-result-2.3.json | 28 +++++++++ .../data/serve-protocol/build-result-2.6.json | 41 ++++++++++++ .../data/serve-protocol/content-address.json | 26 ++++++++ .../data/serve-protocol/drv-output.json | 4 ++ .../optional-content-address.json | 11 ++++ .../serve-protocol/optional-store-path.json | 4 ++ .../data/serve-protocol/realisation.json | 22 +++++++ .../data/serve-protocol/set.json | 22 +++++++ .../data/serve-protocol/store-path.json | 4 ++ .../data/serve-protocol/string.json | 7 +++ .../unkeyed-valid-path-info-2.3.json | 32 ++++++++++ .../unkeyed-valid-path-info-2.4.json | 45 +++++++++++++ .../data/serve-protocol/vector.json | 22 +++++++ .../data/worker-protocol/build-mode.json | 5 ++ .../worker-protocol/build-result-1.27.json | 28 +++++++++ .../worker-protocol/build-result-1.28.json | 41 ++++++++++++ .../worker-protocol/build-result-1.29.json | 41 ++++++++++++ .../worker-protocol/build-result-1.37.json | 43 +++++++++++++ .../data/worker-protocol/content-address.json | 26 ++++++++ .../worker-protocol/derived-path-1.29.json | 16 +++++ .../worker-protocol/derived-path-1.30.json | 17 +++++ .../data/worker-protocol/drv-output.json | 4 ++ .../keyed-build-result-1.29.json | 27 ++++++++ .../optional-content-address.json | 11 ++++ .../worker-protocol/optional-store-path.json | 4 ++ .../optional-trusted-flag.json | 5 ++ .../data/worker-protocol/realisation.json | 22 +++++++ .../data/worker-protocol/set.json | 22 +++++++ .../data/worker-protocol/store-path.json | 4 ++ .../data/worker-protocol/string.json | 7 +++ .../unkeyed-valid-path-info-1.15.json | 32 ++++++++++ .../worker-protocol/valid-path-info-1.15.json | 35 +++++++++++ .../worker-protocol/valid-path-info-1.16.json | 63 +++++++++++++++++++ .../data/worker-protocol/vector.json | 22 +++++++ src/libstore-tests/serve-protocol.cc | 9 +-- src/libstore-tests/worker-protocol.cc | 7 ++- src/libstore/build-result.cc | 16 +++++ .../include/nix/store/build-result.hh | 1 + src/libstore/include/nix/store/path-info.hh | 1 + src/libstore/include/nix/store/realisation.hh | 1 + src/libstore/include/nix/store/store-api.hh | 6 ++ src/libstore/misc.cc | 17 +++++ src/libstore/path-info.cc | 16 +++++ src/libstore/realisation.cc | 14 ++++- .../nix/util/tests/json-characterization.hh | 42 +++++++++---- 57 files changed, 1054 insertions(+), 34 deletions(-) create mode 100644 src/libstore-tests/data/common-protocol/content-address.json create mode 100644 src/libstore-tests/data/common-protocol/drv-output.json create mode 100644 src/libstore-tests/data/common-protocol/optional-content-address.json create mode 100644 src/libstore-tests/data/common-protocol/optional-store-path.json create mode 100644 src/libstore-tests/data/common-protocol/realisation.json create mode 100644 src/libstore-tests/data/common-protocol/set.json create mode 100644 src/libstore-tests/data/common-protocol/store-path.json create mode 100644 src/libstore-tests/data/common-protocol/string.json create mode 100644 src/libstore-tests/data/common-protocol/vector.json create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.2.json create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.3.json create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.6.json create mode 100644 src/libstore-tests/data/serve-protocol/content-address.json create mode 100644 src/libstore-tests/data/serve-protocol/drv-output.json create mode 100644 src/libstore-tests/data/serve-protocol/optional-content-address.json create mode 100644 src/libstore-tests/data/serve-protocol/optional-store-path.json create mode 100644 src/libstore-tests/data/serve-protocol/realisation.json create mode 100644 src/libstore-tests/data/serve-protocol/set.json create mode 100644 src/libstore-tests/data/serve-protocol/store-path.json create mode 100644 src/libstore-tests/data/serve-protocol/string.json create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json create mode 100644 src/libstore-tests/data/serve-protocol/vector.json create mode 100644 src/libstore-tests/data/worker-protocol/build-mode.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.27.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.28.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.29.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.37.json create mode 100644 src/libstore-tests/data/worker-protocol/content-address.json create mode 100644 src/libstore-tests/data/worker-protocol/derived-path-1.29.json create mode 100644 src/libstore-tests/data/worker-protocol/derived-path-1.30.json create mode 100644 src/libstore-tests/data/worker-protocol/drv-output.json create mode 100644 src/libstore-tests/data/worker-protocol/keyed-build-result-1.29.json create mode 100644 src/libstore-tests/data/worker-protocol/optional-content-address.json create mode 100644 src/libstore-tests/data/worker-protocol/optional-store-path.json create mode 100644 src/libstore-tests/data/worker-protocol/optional-trusted-flag.json create mode 100644 src/libstore-tests/data/worker-protocol/realisation.json create mode 100644 src/libstore-tests/data/worker-protocol/set.json create mode 100644 src/libstore-tests/data/worker-protocol/store-path.json create mode 100644 src/libstore-tests/data/worker-protocol/string.json create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json create mode 100644 src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json create mode 100644 src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json create mode 100644 src/libstore-tests/data/worker-protocol/vector.json diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 5b57c6585..0f774df0e 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -6,6 +6,7 @@ #include "nix/store/tests/libstore.hh" #include "nix/util/tests/characterization.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { @@ -16,12 +17,30 @@ class ProtoTest : public CharacterizationTest std::filesystem::path goldenMaster(std::string_view testStem) const override { - return unitTestData / (std::string{testStem + ".bin"}); + return unitTestData / testStem; } public: Path storeDir = "/nix/store"; StoreDirConfig store{storeDir}; + + /** + * Golden test for `T` JSON reading + */ + template + void readJsonTest(PathView testStem, const T & expected) + { + nix::readJsonTest(*this, testStem, expected); + } + + /** + * Golden test for `T` JSON write + */ + template + void writeJsonTest(PathView testStem, const T & decoded) + { + nix::writeJsonTest(*this, testStem, decoded); + } }; template @@ -34,7 +53,7 @@ public: template void readProtoTest(PathView testStem, typename Proto::Version version, T expected) { - CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) { T got = ({ StringSource from{encoded}; Proto::template Serialise::read( @@ -55,7 +74,7 @@ public: template void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) { - CharacterizationTest::writeTest(testStem, [&]() { + CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() { StringSink to; Proto::template Serialise::write( this->store, @@ -69,14 +88,25 @@ public: } }; -#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ - TEST_F(FIXTURE, NAME##_read) \ - { \ - readProtoTest(STEM, VERSION, VALUE); \ - } \ - TEST_F(FIXTURE, NAME##_write) \ - { \ - writeProtoTest(STEM, VERSION, VALUE); \ +#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_read) \ + { \ + readProtoTest(STEM, VERSION, VALUE); \ + } \ + TEST_F(FIXTURE, NAME##_write) \ + { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_json_read) \ + { \ + readJsonTest(STEM, VALUE); \ + } \ + TEST_F(FIXTURE, NAME##_json_write) \ + { \ + writeJsonTest(STEM, VALUE); \ } } // namespace nix diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 7c40e8cdb..fa676eb7f 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -3,6 +3,7 @@ #include #include +#include "nix/util/json-utils.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/store/build-result.hh" @@ -22,7 +23,7 @@ public: template void readProtoTest(PathView testStem, const T & expected) { - CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) { T got = ({ StringSource from{encoded}; CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = from}); @@ -38,7 +39,7 @@ public: template void writeProtoTest(PathView testStem, const T & decoded) { - CharacterizationTest::writeTest(testStem, [&]() -> std::string { + CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() -> std::string { StringSink to; CommonProto::Serialise::write(store, CommonProto::WriteConn{.to = to}, decoded); return to.s; @@ -54,6 +55,14 @@ public: TEST_F(CommonProtoTest, NAME##_write) \ { \ writeProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_read) \ + { \ + readJsonTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_write) \ + { \ + writeJsonTest(STEM, VALUE); \ } CHARACTERIZATION_TEST( diff --git a/src/libstore-tests/data/common-protocol/content-address.json b/src/libstore-tests/data/common-protocol/content-address.json new file mode 100644 index 000000000..9a0d57154 --- /dev/null +++ b/src/libstore-tests/data/common-protocol/content-address.json @@ -0,0 +1,26 @@ +[ + { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "+Xc9Ll6mcPltwaewrk/BAQ56Y3G5T//wzhKUc0zrYu0=" + }, + "method": "text" + }, + { + "hash": { + "algorithm": "sha1", + "format": "base64", + "hash": "gGemBoenViNZM3hiwqns/Fgzqwo=" + }, + "method": "flat" + }, + { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + } +] diff --git a/src/libstore-tests/data/common-protocol/drv-output.json b/src/libstore-tests/data/common-protocol/drv-output.json new file mode 100644 index 000000000..2668d70c9 --- /dev/null +++ b/src/libstore-tests/data/common-protocol/drv-output.json @@ -0,0 +1,4 @@ +[ + "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux" +] diff --git a/src/libstore-tests/data/common-protocol/optional-content-address.json b/src/libstore-tests/data/common-protocol/optional-content-address.json new file mode 100644 index 000000000..6cdaa59a5 --- /dev/null +++ b/src/libstore-tests/data/common-protocol/optional-content-address.json @@ -0,0 +1,11 @@ +[ + null, + { + "hash": { + "algorithm": "sha1", + "format": "base64", + "hash": "gGemBoenViNZM3hiwqns/Fgzqwo=" + }, + "method": "flat" + } +] diff --git a/src/libstore-tests/data/common-protocol/optional-store-path.json b/src/libstore-tests/data/common-protocol/optional-store-path.json new file mode 100644 index 000000000..58519a4d2 --- /dev/null +++ b/src/libstore-tests/data/common-protocol/optional-store-path.json @@ -0,0 +1,4 @@ +[ + null, + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" +] diff --git a/src/libstore-tests/data/common-protocol/realisation.json b/src/libstore-tests/data/common-protocol/realisation.json new file mode 100644 index 000000000..c784e500d --- /dev/null +++ b/src/libstore-tests/data/common-protocol/realisation.json @@ -0,0 +1,22 @@ +[ + { + "dependentRealisations": {}, + "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf", + "qwer" + ] + }, + { + "dependentRealisations": { + "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" + }, + "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf", + "qwer" + ] + } +] diff --git a/src/libstore-tests/data/common-protocol/set.json b/src/libstore-tests/data/common-protocol/set.json new file mode 100644 index 000000000..acd123082 --- /dev/null +++ b/src/libstore-tests/data/common-protocol/set.json @@ -0,0 +1,22 @@ +[ + [], + [ + "" + ], + [ + "", + "bar", + "foo" + ], + [ + [], + [ + "" + ], + [ + "", + "1", + "2" + ] + ] +] diff --git a/src/libstore-tests/data/common-protocol/store-path.json b/src/libstore-tests/data/common-protocol/store-path.json new file mode 100644 index 000000000..16459245b --- /dev/null +++ b/src/libstore-tests/data/common-protocol/store-path.json @@ -0,0 +1,4 @@ +[ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" +] diff --git a/src/libstore-tests/data/common-protocol/string.json b/src/libstore-tests/data/common-protocol/string.json new file mode 100644 index 000000000..d3db4f3b4 --- /dev/null +++ b/src/libstore-tests/data/common-protocol/string.json @@ -0,0 +1,7 @@ +[ + "", + "hi", + "white rabbit", + "大白兔", + "oh no " +] diff --git a/src/libstore-tests/data/common-protocol/vector.json b/src/libstore-tests/data/common-protocol/vector.json new file mode 100644 index 000000000..2b8cc1b3a --- /dev/null +++ b/src/libstore-tests/data/common-protocol/vector.json @@ -0,0 +1,22 @@ +[ + [], + [ + "" + ], + [ + "", + "foo", + "bar" + ], + [ + [], + [ + "" + ], + [ + "", + "1", + "2" + ] + ] +] diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.2.json b/src/libstore-tests/data/serve-protocol/build-result-2.2.json new file mode 100644 index 000000000..b52b6fad5 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.2.json @@ -0,0 +1,28 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "not deterministic", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "builtOutputs": {}, + "startTime": 0, + "status": "built", + "stopTime": 0, + "success": true, + "timesBuilt": 0 + } +] diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.3.json b/src/libstore-tests/data/serve-protocol/build-result-2.3.json new file mode 100644 index 000000000..66b3ca082 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.3.json @@ -0,0 +1,28 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "not deterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": {}, + "startTime": 30, + "status": "built", + "stopTime": 50, + "success": true, + "timesBuilt": 0 + } +] diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.6.json b/src/libstore-tests/data/serve-protocol/build-result-2.6.json new file mode 100644 index 000000000..b64f1109d --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.6.json @@ -0,0 +1,41 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "not deterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/content-address.json b/src/libstore-tests/data/serve-protocol/content-address.json new file mode 100644 index 000000000..9a0d57154 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/content-address.json @@ -0,0 +1,26 @@ +[ + { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "+Xc9Ll6mcPltwaewrk/BAQ56Y3G5T//wzhKUc0zrYu0=" + }, + "method": "text" + }, + { + "hash": { + "algorithm": "sha1", + "format": "base64", + "hash": "gGemBoenViNZM3hiwqns/Fgzqwo=" + }, + "method": "flat" + }, + { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + } +] diff --git a/src/libstore-tests/data/serve-protocol/drv-output.json b/src/libstore-tests/data/serve-protocol/drv-output.json new file mode 100644 index 000000000..2668d70c9 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/drv-output.json @@ -0,0 +1,4 @@ +[ + "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux" +] diff --git a/src/libstore-tests/data/serve-protocol/optional-content-address.json b/src/libstore-tests/data/serve-protocol/optional-content-address.json new file mode 100644 index 000000000..6cdaa59a5 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/optional-content-address.json @@ -0,0 +1,11 @@ +[ + null, + { + "hash": { + "algorithm": "sha1", + "format": "base64", + "hash": "gGemBoenViNZM3hiwqns/Fgzqwo=" + }, + "method": "flat" + } +] diff --git a/src/libstore-tests/data/serve-protocol/optional-store-path.json b/src/libstore-tests/data/serve-protocol/optional-store-path.json new file mode 100644 index 000000000..58519a4d2 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/optional-store-path.json @@ -0,0 +1,4 @@ +[ + null, + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" +] diff --git a/src/libstore-tests/data/serve-protocol/realisation.json b/src/libstore-tests/data/serve-protocol/realisation.json new file mode 100644 index 000000000..c784e500d --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/realisation.json @@ -0,0 +1,22 @@ +[ + { + "dependentRealisations": {}, + "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf", + "qwer" + ] + }, + { + "dependentRealisations": { + "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" + }, + "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf", + "qwer" + ] + } +] diff --git a/src/libstore-tests/data/serve-protocol/set.json b/src/libstore-tests/data/serve-protocol/set.json new file mode 100644 index 000000000..acd123082 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/set.json @@ -0,0 +1,22 @@ +[ + [], + [ + "" + ], + [ + "", + "bar", + "foo" + ], + [ + [], + [ + "" + ], + [ + "", + "1", + "2" + ] + ] +] diff --git a/src/libstore-tests/data/serve-protocol/store-path.json b/src/libstore-tests/data/serve-protocol/store-path.json new file mode 100644 index 000000000..16459245b --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/store-path.json @@ -0,0 +1,4 @@ +[ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" +] diff --git a/src/libstore-tests/data/serve-protocol/string.json b/src/libstore-tests/data/serve-protocol/string.json new file mode 100644 index 000000000..d3db4f3b4 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/string.json @@ -0,0 +1,7 @@ +[ + "", + "hi", + "white rabbit", + "大白兔", + "oh no " +] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json new file mode 100644 index 000000000..3b532c0e8 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json @@ -0,0 +1,32 @@ +[ + { + "ca": null, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + }, + "narSize": 34878, + "references": [], + "registrationTime": null, + "signatures": [], + "ultimate": false + }, + { + "ca": null, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + }, + "narSize": 34878, + "references": [ + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv" + ], + "registrationTime": null, + "signatures": [], + "ultimate": false + } +] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json new file mode 100644 index 000000000..07de33932 --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json @@ -0,0 +1,45 @@ +[ + { + "ca": null, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "references": [ + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv" + ], + "registrationTime": null, + "signatures": [], + "ultimate": false + }, + { + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": null, + "signatures": [ + "fake-sig-1", + "fake-sig-2" + ], + "ultimate": false + } +] diff --git a/src/libstore-tests/data/serve-protocol/vector.json b/src/libstore-tests/data/serve-protocol/vector.json new file mode 100644 index 000000000..2b8cc1b3a --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/vector.json @@ -0,0 +1,22 @@ +[ + [], + [ + "" + ], + [ + "", + "foo", + "bar" + ], + [ + [], + [ + "" + ], + [ + "", + "1", + "2" + ] + ] +] diff --git a/src/libstore-tests/data/worker-protocol/build-mode.json b/src/libstore-tests/data/worker-protocol/build-mode.json new file mode 100644 index 000000000..2ef158f87 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-mode.json @@ -0,0 +1,5 @@ +[ + 0, + 1, + 2 +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.27.json b/src/libstore-tests/data/worker-protocol/build-result-1.27.json new file mode 100644 index 000000000..b52b6fad5 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.27.json @@ -0,0 +1,28 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "not deterministic", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "builtOutputs": {}, + "startTime": 0, + "status": "built", + "stopTime": 0, + "success": true, + "timesBuilt": 0 + } +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.28.json b/src/libstore-tests/data/worker-protocol/build-result-1.28.json new file mode 100644 index 000000000..ce6507452 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.28.json @@ -0,0 +1,41 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "not deterministic", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 0, + "status": "built", + "stopTime": 0, + "success": true, + "timesBuilt": 0 + } +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29.json b/src/libstore-tests/data/worker-protocol/build-result-1.29.json new file mode 100644 index 000000000..b64f1109d --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.29.json @@ -0,0 +1,41 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "not deterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.37.json b/src/libstore-tests/data/worker-protocol/build-result-1.37.json new file mode 100644 index 000000000..7edc02f44 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.37.json @@ -0,0 +1,43 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "not deterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/content-address.json b/src/libstore-tests/data/worker-protocol/content-address.json new file mode 100644 index 000000000..9a0d57154 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/content-address.json @@ -0,0 +1,26 @@ +[ + { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "+Xc9Ll6mcPltwaewrk/BAQ56Y3G5T//wzhKUc0zrYu0=" + }, + "method": "text" + }, + { + "hash": { + "algorithm": "sha1", + "format": "base64", + "hash": "gGemBoenViNZM3hiwqns/Fgzqwo=" + }, + "method": "flat" + }, + { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + } +] diff --git a/src/libstore-tests/data/worker-protocol/derived-path-1.29.json b/src/libstore-tests/data/worker-protocol/derived-path-1.29.json new file mode 100644 index 000000000..f0efe4a35 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/derived-path-1.29.json @@ -0,0 +1,16 @@ +[ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputs": [ + "*" + ] + }, + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputs": [ + "x", + "y" + ] + } +] diff --git a/src/libstore-tests/data/worker-protocol/derived-path-1.30.json b/src/libstore-tests/data/worker-protocol/derived-path-1.30.json new file mode 100644 index 000000000..7a67e4761 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/derived-path-1.30.json @@ -0,0 +1,17 @@ +[ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputs": [ + "*" + ] + }, + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputs": [ + "x", + "y" + ] + } +] diff --git a/src/libstore-tests/data/worker-protocol/drv-output.json b/src/libstore-tests/data/worker-protocol/drv-output.json new file mode 100644 index 000000000..2668d70c9 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/drv-output.json @@ -0,0 +1,4 @@ +[ + "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux" +] diff --git a/src/libstore-tests/data/worker-protocol/keyed-build-result-1.29.json b/src/libstore-tests/data/worker-protocol/keyed-build-result-1.29.json new file mode 100644 index 000000000..2b23eb08c --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/keyed-build-result-1.29.json @@ -0,0 +1,27 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx", + "startTime": 0, + "status": "output rejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "path": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputs": [ + "out" + ] + }, + "startTime": 30, + "status": "not deterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + } +] diff --git a/src/libstore-tests/data/worker-protocol/optional-content-address.json b/src/libstore-tests/data/worker-protocol/optional-content-address.json new file mode 100644 index 000000000..6cdaa59a5 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/optional-content-address.json @@ -0,0 +1,11 @@ +[ + null, + { + "hash": { + "algorithm": "sha1", + "format": "base64", + "hash": "gGemBoenViNZM3hiwqns/Fgzqwo=" + }, + "method": "flat" + } +] diff --git a/src/libstore-tests/data/worker-protocol/optional-store-path.json b/src/libstore-tests/data/worker-protocol/optional-store-path.json new file mode 100644 index 000000000..58519a4d2 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/optional-store-path.json @@ -0,0 +1,4 @@ +[ + null, + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" +] diff --git a/src/libstore-tests/data/worker-protocol/optional-trusted-flag.json b/src/libstore-tests/data/worker-protocol/optional-trusted-flag.json new file mode 100644 index 000000000..2f3c092f8 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/optional-trusted-flag.json @@ -0,0 +1,5 @@ +[ + null, + true, + false +] diff --git a/src/libstore-tests/data/worker-protocol/realisation.json b/src/libstore-tests/data/worker-protocol/realisation.json new file mode 100644 index 000000000..c784e500d --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/realisation.json @@ -0,0 +1,22 @@ +[ + { + "dependentRealisations": {}, + "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf", + "qwer" + ] + }, + { + "dependentRealisations": { + "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" + }, + "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf", + "qwer" + ] + } +] diff --git a/src/libstore-tests/data/worker-protocol/set.json b/src/libstore-tests/data/worker-protocol/set.json new file mode 100644 index 000000000..acd123082 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/set.json @@ -0,0 +1,22 @@ +[ + [], + [ + "" + ], + [ + "", + "bar", + "foo" + ], + [ + [], + [ + "" + ], + [ + "", + "1", + "2" + ] + ] +] diff --git a/src/libstore-tests/data/worker-protocol/store-path.json b/src/libstore-tests/data/worker-protocol/store-path.json new file mode 100644 index 000000000..16459245b --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/store-path.json @@ -0,0 +1,4 @@ +[ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" +] diff --git a/src/libstore-tests/data/worker-protocol/string.json b/src/libstore-tests/data/worker-protocol/string.json new file mode 100644 index 000000000..d3db4f3b4 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/string.json @@ -0,0 +1,7 @@ +[ + "", + "hi", + "white rabbit", + "大白兔", + "oh no " +] diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json new file mode 100644 index 000000000..b123a5b8f --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json @@ -0,0 +1,32 @@ +[ + { + "ca": null, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "references": [], + "registrationTime": 23423, + "signatures": [], + "ultimate": false + }, + { + "ca": null, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "references": [ + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv" + ], + "registrationTime": 23423, + "signatures": [], + "ultimate": false + } +] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json new file mode 100644 index 000000000..9e398e82e --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json @@ -0,0 +1,35 @@ +[ + { + "ca": null, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "references": [], + "registrationTime": 23423, + "signatures": [], + "ultimate": false + }, + { + "ca": null, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo" + ], + "registrationTime": 23423, + "signatures": [], + "ultimate": false + } +] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json new file mode 100644 index 000000000..339ef17c0 --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json @@ -0,0 +1,63 @@ +[ + { + "ca": null, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "references": [], + "registrationTime": 23423, + "signatures": [], + "ultimate": true + }, + { + "ca": null, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo" + ], + "registrationTime": 23423, + "signatures": [ + "fake-sig-1", + "fake-sig-2" + ], + "ultimate": false + }, + { + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=" + }, + "narSize": 34878, + "path": "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [], + "ultimate": false + } +] diff --git a/src/libstore-tests/data/worker-protocol/vector.json b/src/libstore-tests/data/worker-protocol/vector.json new file mode 100644 index 000000000..2b8cc1b3a --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/vector.json @@ -0,0 +1,22 @@ +[ + [], + [ + "" + ], + [ + "", + "foo", + "bar" + ], + [ + [], + [ + "" + ], + [ + "", + "1", + "2" + ] + ] +] diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index a7b69821c..8d26c8241 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -4,6 +4,7 @@ #include #include +#include "nix/util/json-utils.hh" #include "nix/store/serve-protocol.hh" #include "nix/store/serve-protocol-impl.hh" #include "nix/store/serve-protocol-connection.hh" @@ -334,7 +335,7 @@ VERSIONED_CHARACTERIZATION_TEST( }), })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_1, "build-options-2.1", @@ -344,7 +345,7 @@ VERSIONED_CHARACTERIZATION_TEST( .buildTimeout = 6, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_2, "build-options-2.2", @@ -355,7 +356,7 @@ VERSIONED_CHARACTERIZATION_TEST( .maxLogSize = 7, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_3, "build-options-2.3", @@ -368,7 +369,7 @@ VERSIONED_CHARACTERIZATION_TEST( .enforceDeterminism = true, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_7, "build-options-2.7", diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 8f70e937b..f1b639c77 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -4,6 +4,7 @@ #include #include +#include "nix/util/json-utils.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" @@ -649,7 +650,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST_NO_JSON( WorkerProtoTest, clientHandshakeInfo_1_30, "client-handshake-info_1_30", @@ -658,7 +659,7 @@ VERSIONED_CHARACTERIZATION_TEST( {}, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST_NO_JSON( WorkerProtoTest, clientHandshakeInfo_1_33, "client-handshake-info_1_33", @@ -672,7 +673,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST_NO_JSON( WorkerProtoTest, clientHandshakeInfo_1_35, "client-handshake-info_1_35", diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index e3d9e9085..f4bc8ab33 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -153,4 +153,20 @@ BuildResult adl_serializer::from_json(const json & _json) return br; } +KeyedBuildResult adl_serializer::from_json(const json & json0) +{ + auto json = getObject(json0); + + return KeyedBuildResult{ + adl_serializer::from_json(json0), + valueAt(json, "path"), + }; +} + +void adl_serializer::to_json(json & json, const KeyedBuildResult & kbr) +{ + adl_serializer::to_json(json, kbr); + json["path"] = kbr.path; +} + } // namespace nlohmann diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index 4739232f8..96134791b 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -178,3 +178,4 @@ struct KeyedBuildResult : BuildResult } // namespace nix JSON_IMPL(nix::BuildResult) +JSON_IMPL(nix::KeyedBuildResult) diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 8f6115b73..a64e8458d 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -197,3 +197,4 @@ using ValidPathInfos = std::map; } // namespace nix JSON_IMPL(nix::UnkeyedValidPathInfo) +JSON_IMPL(nix::ValidPathInfo) diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index e8a71862e..af0e4aefd 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -182,5 +182,6 @@ public: } // namespace nix +JSON_IMPL(nix::DrvOutput) JSON_IMPL(nix::UnkeyedRealisation) JSON_IMPL(nix::Realisation) diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index c57eff1f0..4c0b156fa 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -1004,4 +1004,10 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr); +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::TrustedFlag) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 8b2a7287e..ea0c8a3c2 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -10,6 +10,7 @@ #include "nix/util/closure.hh" #include "nix/store/filetransfer.hh" #include "nix/util/strings.hh" +#include "nix/util/json-utils.hh" #include @@ -479,3 +480,19 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +TrustedFlag adl_serializer::from_json(const json & json) +{ + return getBoolean(json) ? TrustedFlag::Trusted : TrustedFlag::NotTrusted; +} + +void adl_serializer::to_json(json & json, const TrustedFlag & trustedFlag) +{ + json = static_cast(trustedFlag); +} + +} // namespace nlohmann diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 79c1dc8ac..838bab19f 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -249,4 +249,20 @@ void adl_serializer::to_json(json & json, const UnkeyedVal json = c.toJSON(nullptr, true); } +ValidPathInfo adl_serializer::from_json(const json & json0) +{ + auto json = getObject(json0); + + return ValidPathInfo{ + valueAt(json, "path"), + adl_serializer::from_json(json0), + }; +} + +void adl_serializer::to_json(json & json, const ValidPathInfo & v) +{ + adl_serializer::to_json(json, v); + json["path"] = v.path; +} + } // namespace nlohmann diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index a7f3b98d6..4aeb05874 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -144,6 +144,16 @@ namespace nlohmann { using namespace nix; +DrvOutput adl_serializer::from_json(const json & json) +{ + return DrvOutput::parse(getString(json)); +} + +void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) +{ + json = drvOutput.to_string(); +} + UnkeyedRealisation adl_serializer::from_json(const json & json0) { auto json = getObject(json0); @@ -182,14 +192,14 @@ Realisation adl_serializer::from_json(const json & json0) return Realisation{ static_cast(json0), - DrvOutput::parse(valueAt(json, "id")), + valueAt(json, "id"), }; } void adl_serializer::to_json(json & json, const Realisation & r) { json = static_cast(r); - json["id"] = r.id.to_string(); + json["id"] = r.id; } } // namespace nlohmann diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index d713c615b..0ee6fd2fd 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -11,6 +11,34 @@ namespace nix { +/** + * Golden test for JSON reading + */ +template +void readJsonTest(CharacterizationTest & test, PathView testStem, const T & expected, auto... args) +{ + using namespace nlohmann; + test.readTest(Path{testStem} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + T decoded = adl_serializer::from_json(encoded, args...); + ASSERT_EQ(decoded, expected); + }); +} + +/** + * Golden test for JSON writing + */ +template +void writeJsonTest(CharacterizationTest & test, PathView testStem, const T & value) +{ + using namespace nlohmann; + test.writeTest( + Path{testStem} + ".json", + [&]() -> json { return static_cast(value); }, + [](const auto & file) { return json::parse(readFile(file)); }, + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); +} + /** * Mixin class for writing characterization tests for `nlohmann::json` * conversions for a given type. @@ -26,12 +54,7 @@ struct JsonCharacterizationTest : virtual CharacterizationTest */ void readJsonTest(PathView testStem, const T & expected, auto... args) { - using namespace nlohmann; - readTest(Path{testStem} + ".json", [&](const auto & encodedRaw) { - auto encoded = json::parse(encodedRaw); - T decoded = adl_serializer::from_json(encoded, args...); - ASSERT_EQ(decoded, expected); - }); + nix::readJsonTest(*this, testStem, expected, args...); } /** @@ -42,12 +65,7 @@ struct JsonCharacterizationTest : virtual CharacterizationTest */ void writeJsonTest(PathView testStem, const T & value) { - using namespace nlohmann; - writeTest( - Path{testStem} + ".json", - [&]() -> json { return static_cast(value); }, - [](const auto & file) { return json::parse(readFile(file)); }, - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); + nix::writeJsonTest(*this, testStem, value); } }; From 12760756bb5741e8b5de3d2afb0612ca925b4b62 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Sep 2025 00:29:21 -0400 Subject: [PATCH 08/12] `nlohmann::json` instance and JSON Schema for `MemorySourceAccessor` As documented, this for file system objects themselves, since `MemorySourceAccessor` is an implementation detail. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/file-system-object.md | 21 +++ .../json/fixup-json-schema-generated-doc.sed | 1 + doc/manual/source/protocols/json/meson.build | 1 + .../json/schema/file-system-object-v1 | 1 + .../json/schema/file-system-object-v1.yaml | 65 +++++++++ src/json-schema-checks/file-system-object | 1 + src/json-schema-checks/meson.build | 8 ++ src/json-schema-checks/package.nix | 1 + src/libstore-tests/references.cc | 11 +- .../data/memory-source-accessor/complex.json | 24 ++++ .../data/memory-source-accessor/simple.json | 5 + src/libutil-tests/git.cc | 41 +----- src/libutil-tests/memory-source-accessor.cc | 116 ++++++++++++++++ src/libutil-tests/meson.build | 1 + src/libutil/include/nix/util/bytes.hh | 41 ++++++ .../nix/util/memory-source-accessor.hh | 39 +++++- src/libutil/include/nix/util/meson.build | 1 + src/libutil/memory-source-accessor.cc | 124 +++++++++++++++++- 20 files changed, 456 insertions(+), 48 deletions(-) create mode 100644 doc/manual/source/protocols/json/file-system-object.md create mode 120000 doc/manual/source/protocols/json/schema/file-system-object-v1 create mode 100644 doc/manual/source/protocols/json/schema/file-system-object-v1.yaml create mode 120000 src/json-schema-checks/file-system-object create mode 100644 src/libutil-tests/data/memory-source-accessor/complex.json create mode 100644 src/libutil-tests/data/memory-source-accessor/simple.json create mode 100644 src/libutil-tests/memory-source-accessor.cc create mode 100644 src/libutil/include/nix/util/bytes.hh diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 81061b7a1..3f4b00049 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -37,6 +37,7 @@ mkMesonDerivation (finalAttrs: { (fileset.unions [ ../../.version # For example JSON + ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5be3d6a90..62d59aff6 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -120,6 +120,7 @@ - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) + - [File System Object](protocols/json/file-system-object.md) - [Hash](protocols/json/hash.md) - [Content Address](protocols/json/content-address.md) - [Store Path](protocols/json/store-path.md) diff --git a/doc/manual/source/protocols/json/file-system-object.md b/doc/manual/source/protocols/json/file-system-object.md new file mode 100644 index 000000000..7a93b65ab --- /dev/null +++ b/doc/manual/source/protocols/json/file-system-object.md @@ -0,0 +1,21 @@ +{{#include file-system-object-v1-fixed.md}} + +## Examples + +### Simple + +```json +{{#include schema/file-system-object-v1/simple.json}} +``` + +### Complex + +```json +{{#include schema/file-system-object-v1/complex.json}} +``` + + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 27895d42a..9f7b750da 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -11,6 +11,7 @@ s/\\`/`/g # # As we have more such relative links, more replacements of this nature # should appear below. +s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c56de49c7..fcc0e5ece 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -9,6 +9,7 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false) json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ + 'file-system-object-v1', 'hash-v1', 'content-address-v1', 'store-path-v1', diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1 b/doc/manual/source/protocols/json/schema/file-system-object-v1 new file mode 120000 index 000000000..cbb21a10d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1 @@ -0,0 +1 @@ +../../../../../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml new file mode 100644 index 000000000..c7154b18d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml @@ -0,0 +1,65 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json +title: File System Object +description: | + This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md). + + The schema is recursive because file system objects contain other file system objects. +type: object +required: ["type"] +properties: + type: + type: string + enum: ["regular", "symlink", "directory"] + +# Enforce conditional structure based on `type` +anyOf: + - $ref: "#/$defs/regular" + required: ["type", "contents"] + + - $ref: "#/$defs/symlink" + required: ["type", "target"] + + - $ref: "#/$defs/directory" + required: ["type", "contents"] + +"$defs": + regular: + title: Regular File + required: ["contents"] + properties: + type: + const: "regular" + contents: + type: string + description: Base64-encoded file contents + executable: + type: boolean + description: Whether the file is executable. + default: false + additionalProperties: false + + symlink: + title: Symbolic Link + required: ["target"] + properties: + type: + const: "symlink" + target: + type: string + description: Target path of the symlink. + additionalProperties: false + + directory: + title: Directory + required: ["contents"] + properties: + type: + const: "directory" + contents: + type: object + description: | + Map of names to nested file system objects (for type=directory) + additionalProperties: + $ref: "#" + additionalProperties: false diff --git a/src/json-schema-checks/file-system-object b/src/json-schema-checks/file-system-object new file mode 120000 index 000000000..b26e030c9 --- /dev/null +++ b/src/json-schema-checks/file-system-object @@ -0,0 +1 @@ +../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 65a2651b7..d31ba3d53 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema' # Get all example files schemas = [ + { + 'stem' : 'file-system-object', + 'schema' : schema_dir / 'file-system-object-v1.yaml', + 'files' : [ + 'simple.json', + 'complex.json', + ], + }, { 'stem' : 'hash', 'schema' : schema_dir / 'hash-v1.yaml', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 5365fe75e..2ca43439f 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: { fileset = lib.fileset.unions [ ../../.version ../../doc/manual/source/protocols/json/schema + ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index 9cecd573e..e23b443fb 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -1,5 +1,6 @@ #include "nix/store/references.hh" #include "nix/store/path-references.hh" +#include "nix/util/bytes.hh" #include "nix/util/memory-source-accessor.hh" #include @@ -104,21 +105,21 @@ TEST(references, scanForReferencesDeep) // file1.txt: contains hash1 "file1.txt", File::Regular{ - .contents = "This file references " + hash1 + " in its content", + .contents = to_owned(as_bytes("This file references " + hash1 + " in its content")), }, }, { // file2.txt: contains hash2 and hash3 "file2.txt", File::Regular{ - .contents = "Multiple refs: " + hash2 + " and also " + hash3, + .contents = to_owned(as_bytes("Multiple refs: " + hash2 + " and also " + hash3)), }, }, { // file3.txt: contains no references "file3.txt", File::Regular{ - .contents = "This file has no store path references at all", + .contents = to_owned(as_bytes("This file has no store path references at all")), }, }, { @@ -130,7 +131,7 @@ TEST(references, scanForReferencesDeep) // subdir/file4.txt: contains hash1 again "file4.txt", File::Regular{ - .contents = "Subdirectory file with " + hash1, + .contents = to_owned(as_bytes("Subdirectory file with " + hash1)), }, }, }, @@ -140,7 +141,7 @@ TEST(references, scanForReferencesDeep) // link1: a symlink that contains a reference in its target "link1", File::Symlink{ - .target = hash2 + "-target", + .target = to_owned(as_bytes(hash2 + "-target")), }, }, }, diff --git a/src/libutil-tests/data/memory-source-accessor/complex.json b/src/libutil-tests/data/memory-source-accessor/complex.json new file mode 100644 index 000000000..23c2e5b42 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/complex.json @@ -0,0 +1,24 @@ +{ + "contents": { + "bar": { + "contents": { + "baz": { + "contents": "Z29vZCBkYXksCg==", + "executable": true, + "type": "regular" + }, + "quux": { + "target": "L292ZXIvdGhlcmU=", + "type": "symlink" + } + }, + "type": "directory" + }, + "foo": { + "contents": "aGVsbG8K", + "executable": false, + "type": "regular" + } + }, + "type": "directory" +} diff --git a/src/libutil-tests/data/memory-source-accessor/simple.json b/src/libutil-tests/data/memory-source-accessor/simple.json new file mode 100644 index 000000000..8c61fa730 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/simple.json @@ -0,0 +1,5 @@ +{ + "contents": "YXNkZg==", + "executable": false, + "type": "regular" +} diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index 6180a4cfc..f761c4433 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -224,42 +224,15 @@ TEST_F(GitTest, tree_sha256_write) }); } +namespace memory_source_accessor { + +extern ref exampleComplex(); + +} + TEST_F(GitTest, both_roundrip) { - using File = MemorySourceAccessor::File; - - auto files = make_ref(); - files->root = File::Directory{ - .contents{ - { - "foo", - File::Regular{ - .contents = "hello\n\0\n\tworld!", - }, - }, - { - "bar", - File::Directory{ - .contents = - { - { - "baz", - File::Regular{ - .executable = true, - .contents = "good day,\n\0\n\tworld!", - }, - }, - { - "quux", - File::Symlink{ - .target = "/over/there", - }, - }, - }, - }, - }, - }, - }; + auto files = memory_source_accessor::exampleComplex(); for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) { std::map cas; diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc new file mode 100644 index 000000000..7227c9fa4 --- /dev/null +++ b/src/libutil-tests/memory-source-accessor.cc @@ -0,0 +1,116 @@ +#include + +#include "nix/util/bytes.hh" +#include "nix/util/memory-source-accessor.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +namespace memory_source_accessor { + +using File = MemorySourceAccessor::File; + +ref exampleSimple() +{ + auto sc = make_ref(); + sc->root = File{File::Regular{ + .executable = false, + .contents = to_owned(as_bytes("asdf")), + }}; + return sc; +} + +ref exampleComplex() +{ + auto files = make_ref(); + files->root = File::Directory{ + .contents{ + { + "foo", + File::Regular{ + .contents = to_owned(as_bytes("hello\n\0\n\tworld!")), + }, + }, + { + "bar", + File::Directory{ + .contents = + { + { + "baz", + File::Regular{ + .executable = true, + .contents = to_owned(as_bytes("good day,\n\0\n\tworld!")), + }, + }, + { + "quux", + File::Symlink{ + .target = to_owned(as_bytes("/over/there")), + }, + }, + }, + }, + }, + }, + }; + return files; +} + +} // namespace memory_source_accessor + +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +class MemorySourceAccessorTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(MemorySourceAccessorJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + /* Cannot use `readJsonTest` because need to compare `root` field of + the source accessors for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + auto decoded = static_cast(encoded); + ASSERT_EQ(decoded.root, expected.root); + }); +} + +TEST_P(MemorySourceAccessorJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + MemorySourceAccessorJSON, + MemorySourceAccessorJsonTest, + ::testing::Values( + std::pair{ + "simple", + *memory_source_accessor::exampleSimple(), + }, + std::pair{ + "complex", + *memory_source_accessor::exampleComplex(), + })); + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index c75f4d90a..6c593eef8 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -63,6 +63,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'memory-source-accessor.cc', 'monitorfdhup.cc', 'nix_api_util.cc', 'nix_api_util_internal.cc', diff --git a/src/libutil/include/nix/util/bytes.hh b/src/libutil/include/nix/util/bytes.hh new file mode 100644 index 000000000..7fd6cf8c3 --- /dev/null +++ b/src/libutil/include/nix/util/bytes.hh @@ -0,0 +1,41 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +static inline std::span as_bytes(std::string_view sv) noexcept +{ + return std::span{ + reinterpret_cast(sv.data()), + sv.size(), + }; +} + +static inline std::vector to_owned(std::span bytes) +{ + return std::vector{ + bytes.begin(), + bytes.end(), + }; +} + +/** + * @note this should be avoided, as arbitrary binary data in strings + * views, while allowed, is not really proper. Generally this should + * only be used as a stop-gap with other definitions that themselves + * should be converted to accept `std::span` or + * similar, directly. + */ +static inline std::string_view as_str(std::span sp) +{ + return std::string_view{ + reinterpret_cast(sp.data()), + sp.size(), + }; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index eba282fe1..cd8b3c717 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -4,6 +4,7 @@ #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -25,7 +26,7 @@ struct MemorySourceAccessor : virtual SourceAccessor struct Regular { bool executable = false; - std::string contents; + std::vector contents; bool operator==(const Regular &) const = default; auto operator<=>(const Regular &) const = default; @@ -44,7 +45,7 @@ struct MemorySourceAccessor : virtual SourceAccessor struct Symlink { - std::string target; + std::vector target; bool operator==(const Symlink &) const = default; auto operator<=>(const Symlink &) const = default; @@ -86,7 +87,13 @@ struct MemorySourceAccessor : virtual SourceAccessor */ File * open(const CanonPath & path, std::optional create); - SourcePath addFile(CanonPath path, std::string && contents); + SourcePath addFile(CanonPath path, std::vector && contents); + + /** + * Small wrapper of the other `addFile`, purely for convenience when + * the file in question to be added is a string. + */ + SourcePath addFile(CanonPath path, std::string_view contents); }; inline bool MemorySourceAccessor::File::Directory::operator==( @@ -121,4 +128,30 @@ struct MemorySink : FileSystemObjectSink void createSymlink(const CanonPath & path, const std::string & target) override; }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(MemorySourceAccessor::File::Regular) +JSON_IMPL(MemorySourceAccessor::File::Directory) +JSON_IMPL(MemorySourceAccessor::File::Symlink) +JSON_IMPL(MemorySourceAccessor::File) +JSON_IMPL(MemorySourceAccessor) diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 9a606e15d..89790f908 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -12,6 +12,7 @@ headers = files( 'array-from-string-literal.hh', 'base-n.hh', 'base-nix-32.hh', + 'bytes.hh', 'callback.hh', 'canon-path.hh', 'checked-arithmetic.hh', diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index a9ffb7746..5089470e9 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,4 +1,7 @@ #include "nix/util/memory-source-accessor.hh" +#include "nix/util/base-n.hh" +#include "nix/util/bytes.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -58,7 +61,7 @@ std::string MemorySourceAccessor::readFile(const CanonPath & path) if (!f) throw Error("file '%s' does not exist", path); if (auto * r = std::get_if(&f->raw)) - return r->contents; + return std::string{as_str(r->contents)}; else throw Error("file '%s' is not a regular file", path); } @@ -120,12 +123,12 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path) if (!f) throw Error("file '%s' does not exist", path); if (auto * s = std::get_if(&f->raw)) - return s->target; + return std::string{as_str(s->target)}; else throw Error("file '%s' is not a symbolic link", path); } -SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::vector && contents) { // Create root directory automatically if necessary as a convenience. if (!root && !path.isRoot()) @@ -142,6 +145,11 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents return SourcePath{ref(shared_from_this()), path}; } +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string_view contents) +{ + return addFile(path, to_owned(as_bytes(contents))); +} + using File = MemorySourceAccessor::File; void MemorySink::createDirectory(const CanonPath & path) @@ -190,9 +198,10 @@ void CreateMemoryRegularFile::preallocateContents(uint64_t len) regularFile.contents.reserve(len); } -void CreateMemoryRegularFile::operator()(std::string_view data) +void CreateMemoryRegularFile::operator()(std::string_view data_) { - regularFile.contents += data; + auto data = as_bytes(data_); + regularFile.contents.insert(regularFile.contents.end(), data.begin(), data.end()); } void MemorySink::createSymlink(const CanonPath & path, const std::string & target) @@ -201,7 +210,7 @@ void MemorySink::createSymlink(const CanonPath & path, const std::string & targe if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); if (auto * s = std::get_if(&f->raw)) - s->target = target; + s->target = to_owned(as_bytes(target)); else throw Error("file '%s' is not a symbolic link", path); } @@ -222,3 +231,106 @@ ref makeEmptySourceAccessor() } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +MemorySourceAccessor::File::Regular adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Regular{ + .executable = getBoolean(valueAt(obj, "executable")), + .contents = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "contents"))))), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Regular & val) +{ + json = { + {"executable", val.executable}, + {"contents", base64::encode(val.contents)}, + }; +} + +MemorySourceAccessor::File::Directory +adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Directory{ + .contents = valueAt(obj, "contents"), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Directory & val) +{ + json = { + {"contents", val.contents}, + }; +} + +MemorySourceAccessor::File::Symlink adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Symlink{ + .target = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "target"))))), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Symlink & val) +{ + json = { + {"target", base64::encode(val.target)}, + }; +} + +MemorySourceAccessor::File adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + auto type = getString(valueAt(obj, "type")); + if (type == "regular") + return static_cast(json); + if (type == "directory") + return static_cast(json); + if (type == "symlink") + return static_cast(json); + else + throw Error("unknown type of file '%s'", type); +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor::File & val) +{ + std::visit( + overloaded{ + [&](const MemorySourceAccessor::File::Regular & r) { + json = r; + json["type"] = "regular"; + }, + [&](const MemorySourceAccessor::File::Directory & d) { + json = d; + json["type"] = "directory"; + }, + [&](const MemorySourceAccessor::File::Symlink & s) { + json = s; + json["type"] = "symlink"; + }, + }, + val.raw); +} + +MemorySourceAccessor adl_serializer::from_json(const json & json) +{ + MemorySourceAccessor res; + res.root = json; + return res; +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor & val) +{ + json = val.root; +} + +} // namespace nlohmann From f7dcfd107253ec0b1e8ec8700012c739a0761e07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Sep 2025 02:15:20 -0400 Subject: [PATCH 09/12] JSON impl for `DummyStore` --- .../data/dummy-store/empty.json | 6 ++ .../data/dummy-store/one-derivation.json | 18 ++++ .../data/dummy-store/one-flat-file.json | 35 +++++++ src/libstore-tests/dummy-store.cc | 93 ++++++++++++++++++ src/libstore/dummy-store.cc | 97 +++++++++++++++++++ .../include/nix/store/dummy-store-impl.hh | 10 ++ src/libstore/include/nix/store/dummy-store.hh | 7 ++ 7 files changed, 266 insertions(+) create mode 100644 src/libstore-tests/data/dummy-store/empty.json create mode 100644 src/libstore-tests/data/dummy-store/one-derivation.json create mode 100644 src/libstore-tests/data/dummy-store/one-flat-file.json diff --git a/src/libstore-tests/data/dummy-store/empty.json b/src/libstore-tests/data/dummy-store/empty.json new file mode 100644 index 000000000..976a05843 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/empty.json @@ -0,0 +1,6 @@ +{ + "build-trace": {}, + "contents": {}, + "derivations": {}, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/data/dummy-store/one-derivation.json b/src/libstore-tests/data/dummy-store/one-derivation.json new file mode 100644 index 000000000..201418db6 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-derivation.json @@ -0,0 +1,18 @@ +{ + "build-trace": {}, + "contents": {}, + "derivations": { + "rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": { + "args": [], + "builder": "", + "env": {}, + "inputDrvs": {}, + "inputSrcs": [], + "name": "foo", + "outputs": {}, + "system": "", + "version": 4 + } + }, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json new file mode 100644 index 000000000..d98990baa --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -0,0 +1,35 @@ +{ + "build-trace": {}, + "contents": { + "5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": { + "contents": { + "contents": "asdf", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "method": "nar" + }, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "narSize": 120, + "references": [], + "registrationTime": null, + "signatures": [], + "ultimate": false + } + } + }, + "derivations": {}, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 3dd8137a3..36a0c3019 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -1,11 +1,32 @@ #include +#include +#include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/globals.hh" #include "nix/store/realisation.hh" +#include "nix/util/tests/json-characterization.hh" + namespace nix { +class DummyStoreTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "dummy-store"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + static void SetUpTestSuite() + { + initLibStore(false); + } +}; + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); @@ -35,4 +56,76 @@ TEST(DummyStore, realisation_read) EXPECT_EQ(*value2, value); } +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +struct DummyStoreJsonTest : DummyStoreTest, + JsonCharacterizationTest>, + ::testing::WithParamInterface>> +{}; + +TEST_P(DummyStoreJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + using namespace nlohmann; + /* Cannot use `readJsonTest` because need to dereference the stores + for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + ref decoded = adl_serializer>::from_json(encoded); + ASSERT_EQ(*decoded, *expected); + }); +} + +TEST_P(DummyStoreJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { + initLibStore(false); + auto writeCfg = make_ref(DummyStore::Config::Params{}); + writeCfg->readOnly = false; + return ::testing::Values( + std::pair{ + "empty", + make_ref(DummyStore::Config::Params{})->openDummyStore(), + }, + std::pair{ + "one-flat-file", + [&] { + auto store = writeCfg->openDummyStore(); + store->addToStore( + "my-file", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "asdf", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + return store; + }(), + }, + std::pair{ + "one-derivation", + [&] { + auto store = writeCfg->openDummyStore(); + Derivation drv; + drv.name = "foo"; + store->writeDerivation(drv); + return store; + }(), + }); +}()); + } // namespace nix diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c45a13cc3..218a4e2ed 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -2,6 +2,7 @@ #include "nix/util/archive.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/json-utils.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/realisation.hh" @@ -16,6 +17,16 @@ std::string DummyStoreConfig::doc() ; } +bool DummyStore::PathInfoAndContents::operator==(const PathInfoAndContents & other) const +{ + return info == other.info && contents->root == other.contents->root; +} + +bool DummyStore::operator==(const DummyStore & other) const +{ + return contents == other.contents && buildTrace == other.buildTrace; +} + namespace { class WholeStoreViewAccessor : public SourceAccessor @@ -377,3 +388,89 @@ ref DummyStore::Config::openDummyStore() const static RegisterStoreImplementation regDummyStore; } // namespace nix + +namespace nlohmann { + +using namespace nix; + +DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return DummyStore::PathInfoAndContents{ + .info = valueAt(obj, "info"), + .contents = make_ref(valueAt(obj, "contents")), + }; +} + +void adl_serializer::to_json(json & json, const DummyStore::PathInfoAndContents & val) +{ + json = { + {"info", val.info}, + {"contents", *val.contents}, + }; +} + +ref adl_serializer>::from_json(const json & json) +{ + auto & obj = getObject(json); + ref res = [&] { + auto cfg = make_ref(DummyStore::Config::Params{}); + const_cast(cfg->storeDir_).set(getString(valueAt(obj, "store-dir"))); + cfg->readOnly = true; + return cfg->openDummyStore(); + }(); + for (auto & [k, v] : getObject(valueAt(obj, "contents"))) + res->contents.insert({StorePath{k}, v}); + for (auto & [k, v] : getObject(valueAt(obj, "derivations"))) + res->derivations.insert({StorePath{k}, v}); + for (auto & [k0, v] : getObject(valueAt(obj, "build-trace"))) { + for (auto & [k1, v2] : getObject(v)) { + auto vref = make_ref(v2); + res->buildTrace.insert_or_visit( + { + Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + {{k1, vref}}, + }, + [&](auto & kv) { kv.second.insert_or_assign(k1, vref); }); + } + } + return res; +} + +void adl_serializer>::to_json(json & json, const ref & val) +{ + json = { + {"store-dir", val->storeDir}, + {"contents", + [&] { + auto obj = json::object(); + val->contents.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, + {"derivations", + [&] { + auto obj = json::object(); + val->derivations.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, + {"build-trace", + [&] { + auto obj = json::object(); + val->buildTrace.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + for (auto & [k2, v2] : kv.second) + obj2[k2] = *v2; + }); + return obj; + }()}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 137f81c9b..71869cb7e 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -23,6 +23,8 @@ struct DummyStore : virtual Store { UnkeyedValidPathInfo info; ref contents; + + bool operator==(const PathInfoAndContents &) const; }; /** @@ -54,6 +56,14 @@ struct DummyStore : virtual Store , config(config) { } + + bool operator==(const DummyStore &) const; }; +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::DummyStore::PathInfoAndContents) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index d371c4e51..6b7b20153 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/json-impls.hh" #include @@ -65,4 +66,10 @@ struct DummyStoreConfig : public std::enable_shared_from_this, } }; +template<> +struct json_avoids_null> : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::ref) From f39266353379d4c791a4da4a1ee8c1c098d85f5c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 18:35:19 -0400 Subject: [PATCH 10/12] Remove dependent realisations This progress on #11896. It introduces some issues temporarily which will be fixed when #11928 is fixed. --- .../include/nix/store/tests/protocol.hh | 22 +++-- src/libstore-tests/common-protocol.cc | 50 +++++------ src/libstore-tests/realisation.cc | 81 ++++++++--------- src/libstore-tests/serve-protocol.cc | 12 +-- src/libstore-tests/worker-protocol.cc | 12 +-- src/libstore/build/derivation-goal.cc | 5 -- .../build/drv-output-substitution-goal.cc | 24 ------ src/libstore/ca-specific-schema.sql | 27 ------ src/libstore/include/nix/store/realisation.hh | 16 ---- src/libstore/include/nix/store/store-api.hh | 3 - src/libstore/local-store.cc | 49 ----------- src/libstore/misc.cc | 59 ------------- src/libstore/realisation.cc | 86 +------------------ src/libstore/restricted-store.cc | 2 +- src/libstore/store-api.cc | 40 ++++----- .../ca/duplicate-realisation-in-closure.sh | 5 ++ tests/functional/ca/substitute.sh | 5 +- 17 files changed, 106 insertions(+), 392 deletions(-) diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 0f774df0e..ccb33b17b 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -88,16 +88,22 @@ public: } }; -#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ - TEST_F(FIXTURE, NAME##_read) \ - { \ - readProtoTest(STEM, VERSION, VALUE); \ - } \ - TEST_F(FIXTURE, NAME##_write) \ - { \ - writeProtoTest(STEM, VERSION, VALUE); \ +#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_read) \ + { \ + readProtoTest(STEM, VERSION, VALUE); \ } +#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_write) \ + { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) + #define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ TEST_F(FIXTURE, NAME##_json_read) \ diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index fa676eb7f..6bd7beb44 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -47,24 +47,30 @@ public: } }; -#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ - TEST_F(CommonProtoTest, NAME##_read) \ - { \ - readProtoTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_write) \ - { \ - writeProtoTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_json_read) \ - { \ - readJsonTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_json_write) \ - { \ - writeJsonTest(STEM, VALUE); \ +#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_read) \ + { \ + readProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_read) \ + { \ + readJsonTest(STEM, VALUE); \ } +#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_write) \ + { \ + writeProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_write) \ + { \ + writeJsonTest(STEM, VALUE); \ + } + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) + CHARACTERIZATION_TEST( string, "string", @@ -141,7 +147,7 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +READ_CHARACTERIZATION_TEST( realisation_with_deps, "realisation-with-deps", (std::tuple{ @@ -149,16 +155,6 @@ CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d16049bc5..66f4707e9 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -44,54 +44,45 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } +Realisation simple{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + }, + { + .drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), + .outputName = "foo", + }, +}; + INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, - ([] { - Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - }; - return ::testing::Values( - std::pair{ - "simple", - simple, - }, - std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = {"asdfasdfasdf"}; - return r; - }()}, - std::pair{ - "with-dependent-realisations", - [&] { - auto r = simple; - r.dependentRealisations = {{ - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }}; - return r; - }(), - }); - } + ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = {"asdfasdfasdf"}; + return r; + }(), + })); - ())); +/** + * We no longer have a notion of "dependent realisations", but we still + * want to parse old realisation files. So make this just be a read test + * (no write direction), accordingly. + */ +TEST_F(RealisationTest, dependent_realisations_from_json) +{ + readJsonTest("with-dependent-realisations", simple); +} } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 8d26c8241..afc35d03c 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -116,7 +116,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, realisation_with_deps, "realisation-with-deps", @@ -126,16 +126,6 @@ VERSIONED_CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index f1b639c77..a1f68e231 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -169,7 +169,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, realisation_with_deps, "realisation-with-deps", @@ -179,16 +179,6 @@ VERSIONED_CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 14aa044ea..d8e9ff5e4 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -209,11 +209,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) .outputName = wantedOutput, }}; newRealisation.signatures.clear(); - if (!drv->type().isFixed()) { - auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store; - newRealisation.dependentRealisations = - drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); - } worker.store.signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 8d0a307be..58f3de2b7 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -86,32 +86,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() if (!outputInfo) continue; - bool failed = false; - Goals waitees; - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->config.getHumanReadableURI(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath)); - failed = true; - break; - } - waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId)); - } - } - - if (failed) - continue; - waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); co_await await(std::move(waitees)); diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c5e4e3897..d563b33d8 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -12,30 +12,3 @@ create table if not exists Realisations ( ); create index if not exists IndexRealisations on Realisations(drvPath, outputName); - --- We can end-up in a weird edge-case where a path depends on itself because --- it’s an output of a CA derivation, that happens to be the same as one of its --- dependencies. --- In that case we have a dependency loop (path -> realisation1 -> realisation2 --- -> path) that we need to break by removing the dependencies between the --- realisations -create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict -); --- used by deletion trigger -create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); - --- used by QueryRealisationReferences -create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); --- used by cascade deletion when ValidPaths is deleted -create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index af0e4aefd..3691c14fb 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -56,14 +56,6 @@ struct UnkeyedRealisation StringSet signatures; - /** - * The realisations that are required for the current one to be valid. - * - * When importing this realisation, the store will first check that all its - * dependencies exist, and map to the correct output path - */ - std::map dependentRealisations; - std::string fingerprint(const DrvOutput & key) const; void sign(const DrvOutput & key, const Signer &); @@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation bool isCompatibleWith(const UnkeyedRealisation & other) const; - static std::set closure(Store &, const std::set &); - - static void closure(Store &, const std::set &, std::set & res); - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -154,10 +142,6 @@ struct RealisedPath */ const StorePath & path() const &; - void closure(Store & store, Set & ret) const; - static void closure(Store & store, const Set & startPaths, Set & ret); - Set closure(Store & store) const; - bool operator==(const RealisedPath &) const = default; auto operator<=>(const RealisedPath &) const = default; }; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 4c0b156fa..f272b8588 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -1001,9 +1001,6 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr); - template<> struct json_avoids_null : std::true_type {}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6aeaf0d2..98f2d70af 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -390,21 +390,6 @@ LocalStore::LocalStore(ref config) where drvPath = ? ; )"); - state->stmts->QueryRealisationReferences.create( - state->db, - R"( - select drvPath, outputName from Realisations - join RealisationsRefs on realisationReference = Realisations.id - where referrer = ?; - )"); - state->stmts->AddRealisationReference.create( - state->db, - R"( - insert or replace into RealisationsRefs (referrer, realisationReference) - values ( - (select id from Realisations where drvPath = ? and outputName = ?), - (select id from Realisations where drvPath = ? and outputName = ?)); - )"); } } @@ -654,25 +639,6 @@ void LocalStore::registerDrvOutput(const Realisation & info) concatStringsSep(" ", info.signatures)) .exec(); } - for (auto & [outputId, depPath] : info.dependentRealisations) { - auto localRealisation = queryRealisationCore_(*state, outputId); - if (!localRealisation) - throw Error( - "unable to register the derivation '%s' as it " - "depends on the non existent '%s'", - info.id.to_string(), - outputId.to_string()); - if (localRealisation->second.outPath != depPath) - throw Error( - "unable to register the derivation '%s' as it " - "depends on a realisation of '%s' that doesn’t" - "match what we have locally", - info.id.to_string(), - outputId.to_string()); - state->stmts->AddRealisationReference - .use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName) - .exec(); - } }); } @@ -1609,21 +1575,6 @@ std::optional LocalStore::queryRealisation_(LocalStore return std::nullopt; auto [realisationDbId, res] = *maybeCore; - std::map dependentRealisations; - auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId)); - while (useRealisationRefs.next()) { - auto depId = DrvOutput{ - Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), - useRealisationRefs.getStr(1), - }; - auto dependentRealisation = queryRealisationCore_(state, depId); - assert(dependentRealisation); // Enforced by the db schema - auto outputPath = dependentRealisation->second.outPath; - dependentRealisations.insert({depId, outputPath}); - } - - res.dependentRealisations = dependentRealisations; - return {res}; } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ea0c8a3c2..6829563c2 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -330,65 +330,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } -std::map -drvOutputReferences(const std::set & inputRealisations, const StorePathSet & pathReferences) -{ - std::map res; - - for (const auto & input : inputRealisations) { - if (pathReferences.count(input.outPath)) { - res.insert({input.id, input.outPath}); - } - } - - return res; -} - -std::map -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_) -{ - auto & evalStore = evalStore_ ? *evalStore_ : store; - - std::set inputRealisations; - - auto accumRealisations = [&](this auto & self, - const StorePath & inputDrv, - const DerivedPathMap::ChildNode & inputNode) -> void { - if (!inputNode.value.empty()) { - auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv)); - for (const auto & outputName : inputNode.value) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv)); - DrvOutput key{*outputHash, outputName}; - auto thisRealisation = store.queryRealisation(key); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv)); - inputRealisations.insert({*thisRealisation, std::move(key)}); - } - } - if (!inputNode.value.empty()) { - auto d = makeConstantStorePathRef(inputDrv); - for (const auto & [outputName, childNode] : inputNode.childMap) { - SingleDerivedPath next = SingleDerivedPath::Built{d, outputName}; - self( - // TODO deep resolutions for dynamic derivations, issue #8947, would go here. - resolveDerivedPath(store, next, evalStore_), - childNode); - } - } - }; - - for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) - accumRealisations(inputDrv, inputNode); - - auto info = store.queryPathInfo(outputPath); - - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); -} - OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 4aeb05874..2075b39e6 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,6 +1,5 @@ #include "nix/store/realisation.hh" #include "nix/store/store-api.hh" -#include "nix/util/closure.hh" #include "nix/util/signature/local-keys.hh" #include "nix/util/json-utils.hh" #include @@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const return strHash() + "!" + outputName; } -std::set Realisation::closure(Store & store, const std::set & startOutputs) -{ - std::set res; - Realisation::closure(store, startOutputs, res); - return res; -} - -void Realisation::closure(Store & store, const std::set & startOutputs, std::set & res) -{ - auto getDeps = [&](const Realisation & current) -> std::set { - std::set res; - for (auto & [currentDep, _] : current.dependentRealisations) { - if (auto currentRealisation = store.queryRealisation(currentDep)) - res.insert({*currentRealisation, currentDep}); - else - throw Error("Unrealised derivation '%s'", currentDep.to_string()); - } - return res; - }; - - computeClosure( - startOutputs, - res, - [&](const Realisation & current, std::function> &)> processEdges) { - std::promise> promise; - try { - auto res = getDeps(current); - promise.set_value(res); - } catch (...) { - promise.set_exception(std::current_exception()); - } - return processEdges(promise); - }); -} - std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { nlohmann::json serialized = Realisation{*this, key}; @@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const & bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const { - if (outPath == other.outPath) { - if (dependentRealisations.empty() != other.dependentRealisations.empty()) { - warn( - "Encountered a realisation for '%s' with an empty set of " - "dependencies. This is likely an artifact from an older Nix. " - "I’ll try to fix the realisation if I can", - id.to_string()); - return true; - } else if (dependentRealisations == other.dependentRealisations) { - return true; - } - } - return false; -} - -void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret) -{ - // FIXME: This only builds the store-path closure, not the real realisation - // closure - StorePathSet initialStorePaths, pathsClosure; - for (auto & path : startPaths) - initialStorePaths.insert(path.path()); - store.computeFSClosure(initialStorePaths, pathsClosure); - ret.insert(startPaths.begin(), startPaths.end()); - ret.insert(pathsClosure.begin(), pathsClosure.end()); -} - -void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const -{ - RealisedPath::closure(store, {*this}, ret); -} - -RealisedPath::Set RealisedPath::closure(Store & store) const -{ - RealisedPath::Set ret; - closure(store, ret); - return ret; + return outPath == other.outPath; } } // namespace nix @@ -162,27 +90,19 @@ UnkeyedRealisation adl_serializer::from_json(const json & js if (auto signaturesOpt = optionalValueAt(json, "signatures")) signatures = *signaturesOpt; - std::map dependentRealisations; - if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations")) - for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies)) - dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath}); - return UnkeyedRealisation{ .outPath = valueAt(json, "outPath"), .signatures = signatures, - .dependentRealisations = dependentRealisations, }; } void adl_serializer::to_json(json & json, const UnkeyedRealisation & r) { - auto jsonDependentRealisations = nlohmann::json::object(); - for (auto & [depId, depOutPath] : r.dependentRealisations) - jsonDependentRealisations.emplace(depId.to_string(), depOutPath); json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - {"dependentRealisations", jsonDependentRealisations}, + // back-compat + {"dependentRealisations", json::object()}, }; } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index ef8aaa380..b4d696a53 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -292,7 +292,7 @@ std::vector RestrictedStore::buildPathsWithResults( next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); - for (auto & real : Realisation::closure(*next, newRealisations)) + for (auto & real : newRealisations) goal.addedDrvOutputs.insert(real.id); return results; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c292e2e43..c70af9301 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -915,36 +915,21 @@ std::map copyPaths( SubstituteFlag substitute) { StorePathSet storePaths; - std::set toplevelRealisations; + std::vector realisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto * realisation = std::get_if(&path.raw)) { experimentalFeatureSettings.require(Xp::CaDerivations); - toplevelRealisations.insert(*realisation); + realisations.push_back(realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { - // Copy the realisation closure - processGraph( - Realisation::closure(srcStore, toplevelRealisations), - [&](const Realisation & current) -> std::set { - std::set children; - for (const auto & [drvOutput, _] : current.dependentRealisations) { - auto currentChild = srcStore.queryRealisation(drvOutput); - if (!currentChild) - throw Error( - "incomplete realisation closure: '%s' is a " - "dependency of '%s' but isn't registered", - drvOutput.to_string(), - current.id.to_string()); - children.insert({*currentChild, drvOutput}); - } - return children; - }, - [&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); }); + // Copy the realisations. TODO batch this + for (const auto * realisation : realisations) + dstStore.registerDrvOutput(*realisation, checkSigs); } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want @@ -1055,8 +1040,19 @@ void copyClosure( if (&srcStore == &dstStore) return; - RealisedPath::Set closure; - RealisedPath::closure(srcStore, paths, closure); + StorePathSet closure0; + for (auto & path : paths) { + if (auto * opaquePath = std::get_if(&path.raw)) { + closure0.insert(opaquePath->path); + } + } + + StorePathSet closure1; + srcStore.computeFSClosure(closure0, closure1); + + RealisedPath::Set closure = paths; + for (auto && path : closure1) + closure.insert({std::move(path)}); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index 4a5e8c042..032fb6164 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -25,4 +25,9 @@ nix build -f nondeterministic.nix dep2 --no-link # If everything goes right, we should rebuild dep2 rather than fetch it from # the cache (because that would mean duplicating `current-time` in the closure), # and have `dep1 == dep2`. + +# FIXME: Force the use of small-step resolutions only to fix this in a +# better way (#11896, #11928). +skipTest "temporarily broken because dependent realisations are removed" + nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 9728470f0..2f6ebcef5 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -22,7 +22,10 @@ nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA +# FIXME: `dependentCA` should not need to be explicitly mentioned in +# this. Force the use of small-step resolutions only to allow not +# mentioning it explicitly again. (#11896, #11928). +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA dependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too From b41ce0d263ef4e50181ebb45b3f5dd708e77193c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Feb 2025 23:54:24 -0500 Subject: [PATCH 11/12] Revert "Use the hash modulo in the derivation outputs" Fix #11897 As described in the issue, this makes for a simpler and much more intuitive notion of a realisation key. This is better for pedagogy, and interoperability between more tools. The way the issue was written was that we would switch to only having shallow realisations first, and then do this. But going to only shallow realisations is more complex change, and it turns out we weren't even testing for the benefits that derivation hashes (modulo FODs) provided in the deep realisation case, so I now just want to do this first. Doing this gets the binary cache data structures in order, which will unblock the Hydra fixed-output-derivation tracking work. I don't want to delay that work while I figure out the changes needed for shallow-realisations only. This reverts commit bab1cda0e6c30e25460b5a9c809589d3948f35df. --- src/libcmd/built-path.cc | 18 +- src/libexpr/primops.cc | 23 +- src/libstore-tests/common-protocol.cc | 55 ----- .../data/realisation/simple.json | 12 +- .../data/realisation/with-signature.json | 16 +- .../data/serve-protocol/build-result-2.8.bin | Bin 0 -> 360 bytes .../data/serve-protocol/drv-output-2.8.bin | Bin 0 -> 160 bytes .../data/serve-protocol/realisation-2.8.bin | Bin 0 -> 176 bytes .../unkeyed-realisation-2.8.bin | Bin 0 -> 96 bytes .../worker-protocol/build-result-1.39.bin | Bin 0 -> 424 bytes .../data/worker-protocol/drv-output-1.39.bin | Bin 0 -> 160 bytes .../data/worker-protocol/realisation-1.39.bin | Bin 0 -> 176 bytes .../unkeyed-realisation-1.39.bin | Bin 0 -> 96 bytes src/libstore-tests/dummy-store.cc | 11 +- src/libstore-tests/realisation.cc | 65 ++--- src/libstore-tests/serve-protocol.cc | 140 ++++++----- src/libstore-tests/worker-protocol.cc | 173 +++++++------ src/libstore/binary-cache-store.cc | 12 +- .../build/derivation-building-goal.cc | 7 +- src/libstore/build/derivation-goal.cc | 47 +--- .../build/drv-output-substitution-goal.cc | 7 +- src/libstore/common-protocol.cc | 28 --- src/libstore/daemon.cc | 26 +- src/libstore/derivations.cc | 231 +++++++++++------- src/libstore/dummy-store.cc | 4 +- .../include/nix/store/binary-cache-store.hh | 9 +- .../store/build/derivation-building-misc.hh | 1 - .../nix/store/build/derivation-goal.hh | 2 - .../include/nix/store/common-protocol-impl.hh | 5 +- .../include/nix/store/common-protocol.hh | 7 +- src/libstore/include/nix/store/derivations.hh | 61 +++-- .../include/nix/store/dummy-store-impl.hh | 2 +- .../store/length-prefixed-protocol-helper.hh | 20 +- src/libstore/include/nix/store/realisation.hh | 74 +++--- .../include/nix/store/serve-protocol-impl.hh | 6 +- .../include/nix/store/serve-protocol.hh | 15 +- .../include/nix/store/worker-protocol-impl.hh | 6 +- .../include/nix/store/worker-protocol.hh | 17 +- src/libstore/local-binary-cache-store.cc | 5 +- src/libstore/local-store.cc | 8 +- src/libstore/misc.cc | 9 +- src/libstore/nar-info-disk-cache.cc | 49 ++-- src/libstore/realisation.cc | 85 ++++--- src/libstore/remote-store.cc | 44 +--- src/libstore/restricted-store.cc | 15 +- src/libstore/serve-protocol.cc | 109 ++++++++- src/libstore/store-api.cc | 7 +- src/libstore/unix/build/derivation-builder.cc | 5 +- src/libstore/worker-protocol.cc | 144 ++++++++++- src/nix/build-remote/build-remote.cc | 6 +- src/nix/develop.cc | 10 +- src/perl/lib/Nix/Store.xs | 7 +- tests/functional/ca/substitute.sh | 6 +- 53 files changed, 909 insertions(+), 700 deletions(-) create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/drv-output-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/drv-output-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fc7f18493..f60e4499c 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -108,20 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { - auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath->outPath()), - outputName); - DrvOutput key{*drvOutput, outputName}; + DrvOutput key{ + .drvPath = p.drvPath->outPath(), + .outputName = outputName, + }; auto thisRealisation = store.queryRealisation(key); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(Realisation{*thisRealisation, std::move(key)}); + // We’ve built it, so we must have the realisation. + assert(thisRealisation); + res.insert(Realisation{*thisRealisation, key}); } else { res.insert(outputPath); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d1aae64fa..fb458d0fc 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1718,28 +1718,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{}); } - auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); - switch (hashModulo.kind) { - case DrvHash::Kind::Regular: - for (auto & i : outputs) { - auto h = get(hashModulo.hashes, i); - if (!h) - state.error("derivation produced no hash for output '%s'", i).atPos(v).debugThrow(); - auto outPath = state.store->makeOutputPath(i, *h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign( - i, - DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }); - } - break; - ; - case DrvHash::Kind::Deferred: - for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{}); - } - } + resolveInputAddressed(*state.store, drv); } /* Write the resulting term into the Nix store directory. */ diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 6bd7beb44..71f42517f 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -108,61 +108,6 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( - drvOutput, - "drv-output", - (std::tuple{ - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - })) - -CHARACTERIZATION_TEST( - realisation, - "realisation", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - -READ_CHARACTERIZATION_TEST( - realisation_with_deps, - "realisation-with-deps", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 2ccb1e721..1e4760b56 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,6 +1,10 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } } diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index a28848cb0..4a73ed3ef 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,8 +1,12 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [ - "asdfasdfasdf" - ] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdfasdfasdf" + ] + } } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..fa072599578a351e0f289c1c6bbdc5c1ff90107c GIT binary patch literal 360 zcmZQ&fBjf;Y*hsmSsV}eS+ z%uh-z0*mN_Nd^Y}yvz#y;*$KLRQ+_ra`TKz<3e1tE=(^-E6lvK{Cp6XfgJN-dO-#N E0Qb5q`2YX_ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5be0b15a34567aed217e56dbf2543f2f60a200be GIT binary patch literal 160 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<$PJj|FTB M14ChHX$6Q00QuS{z5oCK literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5295ee34400a58e06c919eaf4b0c70b95b713fc9 GIT binary patch literal 176 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<-kNBm)D9 Z<}olq^|3(d#Nw1R5EI5PEKe;0@c}{FC(r-@ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..10f4ebcb2eb1622e1c56b462f36a7db1e863b345 GIT binary patch literal 96 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6GR&W3zSYQPDukXVf@1K)FKcc E0Dd79Jpcdz literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.39.bin b/src/libstore-tests/data/worker-protocol/build-result-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..11bec3e6e3b7a82f7b2d792d687a807644ff9d05 GIT binary patch literal 424 zcmZQ&fBO)Le|4^xkB4qQI< ztSm?kobS(|0^&1)2nGfQn0t~Ei@+lKV3L7BKQFUFzqlm7C{;h*u-rVO(zp;8tqao& U(h4&#Ek7T`Wgy2qm|lN>LeDBQuy}U`R@=0<$PJj|FTB M14ChHX$6Q00QuS{z5oCK literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation-1.39.bin b/src/libstore-tests/data/worker-protocol/realisation-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..5295ee34400a58e06c919eaf4b0c70b95b713fc9 GIT binary patch literal 176 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<-kNBm)D9 Z<}olq^|3(d#Nw1R5EI5PEKe;0@c}{FC(r-@ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..10f4ebcb2eb1622e1c56b462f36a7db1e863b345 GIT binary patch literal 96 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6GR&W3zSYQPDukXVf@1K)FKcc E0Dd79Jpcdz literal 0 HcmV?d00001 diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 36a0c3019..f53fc03b4 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -37,20 +37,19 @@ TEST(DummyStore, realisation_read) return cfg->openDummyStore(); }(); - auto drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }; - store->buildTrace.insert({drvHash, {{outputName, make_ref(value)}}}); + store->buildTrace.insert({drvPath, {{outputName, make_ref(value)}}}); - auto value2 = store->queryRealisation({drvHash, outputName}); + auto value2 = store->queryRealisation({drvPath, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 66f4707e9..3087bd1b5 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -44,45 +44,30 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } -Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, -}; - -INSTANTIATE_TEST_SUITE_P( - RealisationJSON, - RealisationJsonTest, - ::testing::Values( - std::pair{ - "simple", - simple, - }, - std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = {"asdfasdfasdf"}; - return r; - }(), - })); - -/** - * We no longer have a notion of "dependent realisations", but we still - * want to parse old realisation files. So make this just be a read test - * (no write direction), accordingly. - */ -TEST_F(RealisationTest, dependent_realisations_from_json) -{ - readJsonTest("with-dependent-realisations", simple); -} +INSTANTIATE_TEST_SUITE_P(RealisationJSON, RealisationJsonTest, ([] { + Realisation simple{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", + }, + }; + return ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = {"asdfasdfasdf"}; + return r; + }(), + }); + }())); } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index afc35d03c..1bffc08ef 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -73,16 +73,16 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput, - "drv-output", - defaultVersion, + drvOutput_2_8, + "drv-output-2.8", + 2 << 8 | 8, (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) @@ -91,46 +91,27 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_2_8, + "unkeyed-realisation-2.8", + 2 << 8 | 8, + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_2_8, + "realisation-2.8", + 2 << 8 | 8, + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -180,7 +161,10 @@ VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2 t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully right + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -206,27 +190,65 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, buildResult_2_8, "build-result-2.8", 2 << 8 | 8, ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + }}}, + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index a1f68e231..853568940 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -129,61 +129,42 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output", - defaultVersion, + "drv-output-1.39", + 1 << 8 | 39, (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_1_39, + "unkeyed-realisation-1.39", + 1 << 8 | 39, + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_1_39, + "realisation-1.39", + 1 << 8 | 39, + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -205,7 +186,10 @@ VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully right + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -224,25 +208,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -251,7 +223,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -277,27 +250,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -310,7 +269,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -336,27 +296,60 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, buildResult_1_39, "build-result-1.39", 1 << 8 | 39, ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + }}}, + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 274e47271..6a213da57 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -514,7 +514,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.to_string() + ".doi"; + return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -535,7 +535,10 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); + {}, + "while parsing file '%s' as a build trace value for key '%s'", + outputInfoFilePath, + id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -551,7 +554,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); + upsertFile( + makeRealisationPath(info.id), + static_cast(static_cast(info)).dump(), + "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index c72130142..8e06b2e08 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -212,9 +212,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild() given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; + for (auto & [outputName, _] : drv->outputs) { + InitialOutput v; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -1106,7 +1105,7 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{info.outputHash, i.first}; + auto drvOutput = DrvOutput{drvPath, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d8e9ff5e4..d27485abe 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -36,12 +36,6 @@ DerivationGoal::DerivationGoal( , drvPath(drvPath) , wantedOutput(wantedOutput) , drv{std::make_unique(drv)} - , outputHash{[&] { - auto outputHashes = staticOutputHashes(worker.evalStore, drv); - if (auto * mOutputHash = get(outputHashes, wantedOutput)) - return *mOutputHash; - throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); - }()} , buildMode(buildMode) { @@ -100,7 +94,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (settings.useSubstitutes && drvOptions.substitutesAllowed()) { if (!checkResult) - waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput}))); + waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{drvPath, wantedOutput}))); else { auto * cap = getDerivationCA(*drv); waitees.insert(upcast_goal(worker.makePathSubstitutionGoal( @@ -167,12 +161,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); - - auto outputHash = get(outputHashes, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); - if ((!outputHash) || (!resolvedHash)) + if (!drv->outputs.contains(wantedOutput)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -181,7 +170,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return static_cast(*take1); + return *take1; /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -189,7 +178,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvHash = *resolvedHash, + .drvPath = pathResolved, .outputName = wantedOutput, }); if (take2) @@ -205,7 +194,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) Realisation newRealisation{ realisation, { - .drvHash = *outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }}; newRealisation.signatures.clear(); @@ -251,16 +240,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -385,7 +365,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{outputHash, wantedOutput}; + auto drvOutput = DrvOutput{drvPath, wantedOutput}; std::optional mRealisation; @@ -425,7 +405,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvHash = outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }, }); @@ -448,16 +428,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke { buildResult.inner = BuildResult::Success{ .status = status, - .builtOutputs = {{ - wantedOutput, - { - std::move(builtOutput), - DrvOutput{ - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, }; mcExpectedBuilds.reset(); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 58f3de2b7..34a263edd 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -12,7 +12,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.to_string()); + name = fmt("substitution of '%s'", id.render(worker.store)); trace("created"); } @@ -107,7 +107,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug( + "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); if (substituterFailed) { worker.failedSubstitutions++; @@ -122,7 +123,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.to_string()); + return "a$" + std::string(id.render(worker.store)); } void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index b069c9498..6b819fd6e 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -46,34 +46,6 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - try { - return nlohmann::json::parse(rawInput); - } catch (Error & e) { - e.addTrace({}, "while parsing a realisation object in the remote protocol"); - throw; - } -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << static_cast(realisation).dump(); -} - -DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 937946134..0b494d0d7 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -963,33 +963,31 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { - auto outputId = DrvOutput::parse(readString(conn.from)); - auto outputPath = StorePath(readString(conn.from)); - store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); - } else { - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); - } + // TODO move to WorkerProto::Serialise and friends + // if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) { + // throw Error("old-style build traces no longer supported"); + //} + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); logger->stopWork(); break; } case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(conn.from)); - auto info = store->queryRealisation(outputId); + auto outputId = WorkerProto::Serialise::read(*store, rconn); + std::optional info = *store->queryRealisation(outputId); logger->stopWork(); if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); WorkerProto::write(*store, wconn, outPaths); + } else if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) { + // No longer support this format + WorkerProto::write(*store, wconn, StringSet{}); } else { - std::set realisations; - if (info) - realisations.insert({*info, outputId}); - WorkerProto::write(*store, wconn, realisations); + WorkerProto::write(*store, wconn, info); } break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index e6ac08fd9..b9ec4d64b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -866,13 +866,14 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) +static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); + // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -895,12 +896,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto type = drv.type(); - /* Return a fixed hash for fixed-output derivations. */ - if (type.isFixed()) { + if (drv.type().isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -910,54 +909,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return DrvHash{ - .hashes = outputHashes, - .kind = DrvHash::Kind::Regular, - }; + return outputHashes; } - auto kind = std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return DrvHash::Kind::Regular; - }, - [](const DerivationType::ContentAddressed & ca) { - return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; - }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, - drv.type().raw); + if (std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return false; + }, + [](const DerivationType::ContentAddressed & ca) { + // Already covered + assert(!ca.fixed); + return true; + }, + [](const DerivationType::Impure &) { return true; }}, + drv.type().raw)) { + return DrvHashModulo::DeferredDrv{}; + } + /* For other derivations, replace the inputs paths with recursive + calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { + /* Need to build and resolve dynamic derivations first */ + if (!node.childMap.empty()) { + return DrvHashModulo::DeferredDrv{}; + } + const auto & res = pathDerivationModulo(store, drvPath); - if (res.kind == DrvHash::Kind::Deferred) - kind = DrvHash::Kind::Deferred; - for (auto & outputName : node.value) { - const auto h = get(res.hashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); + if (std::visit( + overloaded{ + [&](const DrvHashModulo::DeferredDrv &) { return true; }, + // Regular non-CA derivation, replace derivation + [&](const DrvHashModulo::DrvHash & drvHash) { + inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + return false; + }, + // CA derivation's output hashes + [&](const DrvHashModulo::CaOutputHashes & outputHashes) { + for (auto & outputName : node.value) { + /* Put each one in with a single "out" output.. */ + const auto h = get(outputHashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2.insert_or_assign( + h->to_string(HashFormat::Base16, false), + DerivedPathMap::ChildNode{ + .value = {"out"}, + }); + } + return false; + }, + }, + res.raw)) { + return DrvHashModulo::DeferredDrv{}; } } - auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); - - std::map outputHashes; - for (const auto & [outputName, _] : drv.outputs) { - outputHashes.insert_or_assign(outputName, hash); - } - - return DrvHash{ - .hashes = outputHashes, - .kind = kind, - }; -} - -std::map staticOutputHashes(Store & store, const Derivation & drv) -{ - return hashDerivationModulo(store, drv, true).hashes; + return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1107,22 +1118,39 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +void resolveInputAddressed(Store & store, Derivation & drv) { - drv.applyRewrites(rewrites); + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, drv, true); + } + return *hashModulo_; + }; - auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative(output.raw)) { - auto h = get(hashModulo.hashes, outputName); - if (!h) - throw Error( - "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); - auto outPath = store.makeOutputPath(outputName, *h, drv.name); - drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }; + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + auto outPath = store.makeOutputPath(outputName, drvHash, drv.name); + drv.env.insert_or_assign(outputName, store.printStorePath(outPath)); + output = DerivationOutput::InputAddressed{ + .path = std::move(outPath), + }; + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + deferred (waiting to be input-addressed). */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + // Nothing to do, already deferred + }, + }, + hashModulo().raw); } } } @@ -1205,9 +1233,13 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - rewriteDerivation(store, resolved, inputRewrites); + resolved.applyRewrites(inputRewrites); - return resolved; + Derivation resolved2{std::move(resolved)}; + + resolveInputAddressed(store, resolved2); + + return resolved2; } void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const @@ -1235,46 +1267,79 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const // combinations that are currently prohibited. type(); - std::optional hashesModulo; - for (auto & i : outputs) { + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, *this, true); + } + return *hashModulo_; + }; + + for (auto & [outputName, output] : outputs) { std::visit( overloaded{ [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(store, *this, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error( - "derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - store.printStorePath(drvPath), - store.printStorePath(doia.path), - i.first); - StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error( - "derivation '%s' has incorrect output '%s', should be '%s'", - store.printStorePath(drvPath), - store.printStorePath(doia.path), - store.printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + StorePath recomputed = store.makeOutputPath(outputName, drvHash, drvName); + if (doia.path != recomputed) + throw Error( + "derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), + store.printStorePath(doia.path), + store.printStorePath(recomputed)); + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed. */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + throw Error( + "derivation '%s' has output '%s', but derivation is not yet ready to be input-addressed", + store.printStorePath(drvPath), + store.printStorePath(doia.path)); + }, + }, + hashModulo().raw); + envHasRightPath(doia.path, outputName); }, [&](const DerivationOutput::CAFixed & dof) { - auto path = dof.path(store, drvName, i.first); - envHasRightPath(path, i.first); + auto path = dof.path(store, drvName, outputName); + envHasRightPath(path, outputName); }, [&](const DerivationOutput::CAFloating &) { /* Nothing to check */ }, [&](const DerivationOutput::Deferred &) { /* Nothing to check */ + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + throw Error( + "derivation '%s' has deferred output '%s', yet is ready to be input-addressed", + store.printStorePath(drvPath), + outputName); + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed. */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + /* Nothing to check */ + }, + }, + hashModulo().raw); }, [&](const DerivationOutput::Impure &) { /* Nothing to check */ }, }, - i.second.raw); + output.raw); } } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 218a4e2ed..2a4c21b5f 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -332,7 +332,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { auto ref = make_ref(output); - buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, ref}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, ref}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, make_ref(output)); }); } @@ -341,7 +341,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(it->second.get_ptr()); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index e64dc3eae..8623d5f4e 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -82,8 +82,13 @@ protected: /** * The prefix under which realisation infos will be stored + * + * @note The previous (still experimental, though) hash-keyed + * realisations were under "realisations". "build trace" is a better + * name anyways (issue #11895), and this serves as some light + * versioning. */ - constexpr const static std::string realisationsPrefix = "realisations"; + constexpr const static std::string realisationsPrefix = "build-trace"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; @@ -92,7 +97,7 @@ protected: /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvOutput}.doi`. + * It's `${realisationsPrefix}/${drvPath}/${outputName}`. */ std::string makeRealisationPath(const DrvOutput & id); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 2b68fa178..8d6892839 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,7 +45,6 @@ struct InitialOutputStatus struct InitialOutput { - Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 0fe610987..c732fd441 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -71,8 +71,6 @@ private: */ std::unique_ptr drv; - const Hash outputHash; - const BuildMode buildMode; /** diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index cb1020a3c..d0eebe0bc 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,12 +26,13 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index c1d22fa6c..45bec3ff2 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -12,7 +12,6 @@ struct Source; class StorePath; struct ContentAddress; struct DrvOutput; -struct Realisation; /** * Shared serializers between the worker protocol, serve protocol, and a @@ -70,8 +69,6 @@ template<> DECLARE_COMMON_SERIALISER(ContentAddress); template<> DECLARE_COMMON_SERIALISER(DrvOutput); -template<> -DECLARE_COMMON_SERIALISER(Realisation); #define COMMA_ , template @@ -81,8 +78,8 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); +template +DECLARE_COMMON_SERIALISER(std::map); #undef COMMA_ /** diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 259314d3f..045ba0eab 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -427,34 +427,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHash +struct DrvHashModulo { /** - * Map from output names to hashes + * Single hash for the derivation + * + * This is for an input-addressed derivation that doesn't + * transitively depend on any floating-CA derivations. */ - std::map hashes; - - enum struct Kind : bool { - /** - * Statically determined derivations. - * This hash will be directly used to compute the output paths - */ - Regular, - - /** - * Floating-output derivations (and their reverse dependencies). - */ - Deferred, - }; + using DrvHash = Hash; /** - * The kind of derivation this is, simplified for just "derivation hash - * modulo" purposes. + * Known CA drv's output hashes, for fixed-output derivations whose + * output hashes are always known since they are fixed up-front. */ - Kind kind; -}; + using CaOutputHashes = std::map; -void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + /** + * This derivation doesn't yet have known output hashes. + * + * Either because itself is floating CA, or it (transtively) depends + * on a floating CA derivation. + */ + using DeferredDrv = std::monostate; + + using Raw = std::variant; + + Raw raw; + + bool operator==(const DrvHashModulo &) const = default; + // auto operator <=> (const DrvHashModulo &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); +}; /** * Returns hashes with the details of fixed-output subderivations @@ -480,15 +485,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * Return a map associating each output to a hash that uniquely identifies its - * derivation (modulo the self-references). + * If a derivation is input addressed and doesn't yet have its input + * addressed (is deferred) try using `hashDerivationModulo`. * - * \todo What is the Hash in this map? + * Does nothing if not deferred input-addressed, or + * `hashDerivationModulo` indicates it is missing inputs' output paths + * and is not yet ready (and must stay deferred). */ -std::map staticOutputHashes(Store & store, const Derivation & drv); +void resolveInputAddressed(Store & store, Derivation & drv); struct DrvHashFct { @@ -503,7 +510,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 71869cb7e..42fa32d61 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -49,7 +49,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map>> buildTrace; + boost::concurrent_flat_map>> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 035019340..e1a80e8dc 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); -#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); +#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map -LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ LengthPrefixedProtoHelper>::read(const StoreDirConfig & st return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 3691c14fb..e5ac6d73d 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -18,33 +18,40 @@ struct OutputsSpec; /** * A general `Realisation` key. * - * This is similar to a `DerivedPath::Opaque`, but the derivation is - * identified by its "hash modulo" instead of by its store path. + * This is similar to a `DerivedPath::Built`, except it is only a single + * step: `drvPath` is a `StorePath` rather than a `DerivedPath`. */ struct DrvOutput { /** - * The hash modulo of the derivation. - * - * Computed from the derivation itself for most types of - * derivations, but computed from the (fixed) content address of the - * output for fixed-output derivations. + * The store path to the derivation */ - Hash drvHash; + StorePath drvPath; /** * The name of the output. */ OutputName outputName; + /** + * Skips the store dir on the `drvPath` + */ std::string to_string() const; - std::string strHash() const - { - return drvHash.to_string(HashFormat::Base16, true); - } + /** + * Skips the store dir on the `drvPath` + */ + static DrvOutput from_string(std::string_view); - static DrvOutput parse(const std::string &); + /** + * Includes the store dir on `drvPath` + */ + std::string render(const StoreDirConfig & store) const; + + /** + * Includes the store dir on `drvPath` + */ + static DrvOutput parse(const StoreDirConfig & store, std::string_view); bool operator==(const DrvOutput &) const = default; auto operator<=>(const DrvOutput &) const = default; @@ -64,6 +71,16 @@ struct UnkeyedRealisation size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; + /** + * Just check the `outPath`. Signatures don't matter for this. + * Callers must ensure that the corresponding key is the same for + * most use-cases. + */ + bool isCompatibleWith(const UnkeyedRealisation & other) const + { + return outPath == other.outPath; + } + const StorePath & getPath() const { return outPath; @@ -77,8 +94,6 @@ struct Realisation : UnkeyedRealisation { DrvOutput id; - bool isCompatibleWith(const UnkeyedRealisation & other) const; - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -89,16 +104,7 @@ struct Realisation : UnkeyedRealisation * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; - -/** - * Collection type for multiple derivations' outputs' `Realisation`s. - * - * `DrvOutput` is used because in general the derivations are not all - * the same, so we need to identify firstly which derivation, and - * secondly which output of that derivation. - */ -typedef std::map DrvOutputs; +typedef std::map SingleDrvOutputs; struct OpaquePath { @@ -149,19 +155,17 @@ struct RealisedPath class MissingRealisation : public Error { public: - MissingRealisation(DrvOutput & outputId) - : MissingRealisation(outputId.outputName, outputId.strHash()) + MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId) + : MissingRealisation(store, outputId.drvPath, outputId.outputName) { } - MissingRealisation(std::string_view drv, OutputName outputName) - : Error( - "cannot operate on output '%s' of the " - "unbuilt derivation '%s'", - outputName, - drv) - { - } + MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName); + MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName); }; } // namespace nix diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index a9617165a..24fc3c9ab 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -34,8 +34,10 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , SERVE_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index 974bf42d5..1b548b517 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION (2 << 8 | 7) +#define SERVE_PROTOCOL_VERSION (2 << 8 | 8) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,6 +18,9 @@ struct Source; // items being serialised struct BuildResult; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; /** * The "serve protocol", used by ssh:// stores. @@ -178,6 +181,12 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> +DECLARE_SERVE_SERIALISER(DrvOutput); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_SERVE_SERIALISER(Realisation); +template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); @@ -190,8 +199,8 @@ DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -template -DECLARE_SERVE_SERIALISER(std::map); +template +DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 26f6b9d44..605663d2e 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -34,8 +34,10 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , WORKER_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 6ae5fdcbc..4374ba7a6 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -12,7 +12,7 @@ namespace nix { /* Note: you generally shouldn't change the protocol version. Define a new `WorkerProto::Feature` instead. */ -#define PROTOCOL_VERSION (1 << 8 | 38) +#define PROTOCOL_VERSION (1 << 8 | 39) #define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -35,6 +35,9 @@ struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; enum BuildMode : uint8_t; enum TrustedFlag : bool; @@ -261,6 +264,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> +DECLARE_WORKER_SERIALISER(DrvOutput); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_WORKER_SERIALISER(Realisation); +template<> +DECLARE_WORKER_SERIALISER(std::optional); +template<> DECLARE_WORKER_SERIALISER(BuildMode); template<> DECLARE_WORKER_SERIALISER(std::optional); @@ -277,8 +288,8 @@ DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -template -DECLARE_WORKER_SERIALISER(std::map); +template +DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 63730a01b..641a397b3 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -56,9 +56,10 @@ protected: void upsertFile( const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) override { - auto path2 = config->binaryCacheDir + "/" + path; + auto path2 = std::filesystem::path{config->binaryCacheDir} / path; static std::atomic counter{0}; - Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); + createDirs(path2.parent_path()); + auto tmp = path2 + fmt(".tmp.%d.%d", getpid(), ++counter); AutoDelete del(tmp, false); writeFile(tmp, source); std::filesystem::rename(tmp, path2); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 98f2d70af..cae2dc4a0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -110,8 +110,6 @@ struct LocalStore::State::Stmts SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; - SQLiteStmt QueryRealisationReferences; - SQLiteStmt AddRealisationReference; }; LocalStore::LocalStore(ref config) @@ -621,7 +619,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto combinedSignatures = oldR->signatures; combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput - .use()(concatStringsSep(" ", combinedSignatures))(info.id.strHash())(info.id.outputName) + .use()(concatStringsSep(" ", combinedSignatures))(info.id.drvPath.to_string())(info.id.outputName) .exec(); } else { throw Error( @@ -635,7 +633,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) } } else { state->stmts->RegisterRealisedOutput - .use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( + .use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))( concatStringsSep(" ", info.signatures)) .exec(); } @@ -1553,7 +1551,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si std::optional> LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id) { - auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName)); + auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName)); if (!useQueryRealisedOutput.next()) return std::nullopt; auto realisationDbId = useQueryRealisedOutput.getInt(0); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6829563c2..b46d900b7 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -240,16 +240,15 @@ MissingPaths Store::queryMissing(const std::vector & targets) // If there are unknown output paths, attempt to find if the // paths are known to substituters through a realisation. - auto outputHashes = staticOutputHashes(*this, *drv); knownOutputPaths = true; - for (auto [outputName, hash] : outputHashes) { + for (auto & [outputName, _] : drv->outputs) { if (!bfd.outputs.contains(outputName)) continue; bool found = false; for (auto & sub : getDefaultSubstituters()) { - auto realisation = sub->queryRealisation({hash, outputName}); + auto realisation = sub->queryRealisation({drvPath, outputName}); if (!realisation) continue; found = true; @@ -362,7 +361,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } @@ -386,7 +385,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); + throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output); return *optPath; }, }, diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 11608a667..116957e55 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -44,10 +44,16 @@ create table if not exists NARs ( create table if not exists Realisations ( cache integer not null, - outputId text not null, - content blob, -- Json serialisation of the realisation, or null if the realisation is absent + + drvPath text not null, + outputName text not null, + + -- The following are null if the realisation is absent + outputPath text, + sigs text, + timestamp integer not null, - primary key (cache, outputId), + primary key (cache, drvPath, outputName), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -121,24 +127,24 @@ public: state->insertRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, content, timestamp) - values (?, ?, ?, ?) + insert or replace into Realisations(cache, drvPath, outputName, outputPath, sigs, timestamp) + values (?, ?, ?, ?, ?, ?) )"); state->insertMissingRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, timestamp) - values (?, ?, ?) + insert or replace into Realisations(cache, drvPath, outputName, timestamp) + values (?, ?, ?, ?) )"); state->queryRealisation.create( state->db, R"( - select content from Realisations - where cache = ? and outputId = ? and - ((content is null and timestamp > ?) or - (content is not null and timestamp > ?)) + select outputPath, sigs from Realisations + where cache = ? and drvPath = ? and outputName = ? and + ((outputPath is null and timestamp > ?) or + (outputPath is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -295,22 +301,27 @@ public: auto now = time(0); - auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( + auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)( now - settings.ttlNegativeNarInfoCache)(now - settings.ttlPositiveNarInfoCache)); if (!queryRealisation.next()) - return {oUnknown, 0}; + return {oUnknown, nullptr}; if (queryRealisation.isNull(0)) - return {oInvalid, 0}; + return {oInvalid, nullptr}; try { return { oValid, - std::make_shared(nlohmann::json::parse(queryRealisation.getStr(0))), + std::make_shared( + UnkeyedRealisation{ + .outPath = StorePath{queryRealisation.getStr(0)}, + .signatures = nlohmann::json::parse(queryRealisation.getStr(1)), + }, + id), }; } catch (Error & e) { - e.addTrace({}, "while parsing the local disk cache"); + e.addTrace({}, "reading build trace key-value from the local disk cache"); throw; } }); @@ -355,7 +366,9 @@ public: auto & cache(getCache(*state, uri)); state->insertRealisation - .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())(time(0)) + .use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)( + realisation.outPath.to_string())(static_cast(realisation.signatures).dump())( + time(0)) .exec(); }); } @@ -366,7 +379,7 @@ public: auto state(_state.lock()); auto & cache(getCache(*state, uri)); - state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec(); + state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(0)).exec(); }); } }; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 2075b39e6..aaad8f662 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -8,28 +8,34 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); -DrvOutput DrvOutput::parse(const std::string & strRep) +DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s) { - size_t n = strRep.find("!"); - if (n == strRep.npos) - throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); - + size_t n = s.rfind('^'); + if (n == s.npos) + throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s); return DrvOutput{ - .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), - .outputName = strRep.substr(n + 1), + .drvPath = store.parseStorePath(s.substr(0, n)), + .outputName = OutputName{s.substr(n + 1)}, }; } +std::string DrvOutput::render(const StoreDirConfig & store) const +{ + return std::string(store.printStorePath(drvPath)) + "^" + outputName; +} + std::string DrvOutput::to_string() const { - return strHash() + "!" + outputName; + return std::string(drvPath.to_string()) + "^" + outputName; } std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { - nlohmann::json serialized = Realisation{*this, key}; - serialized.erase("signatures"); - return serialized.dump(); + auto serialised = static_cast(Realisation{*this, key}); + auto value = serialised.find("value"); + assert(value != serialised.end()); + value->erase("signatures"); + return serialised.dump(); } void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) @@ -61,9 +67,20 @@ const StorePath & RealisedPath::path() const & return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw); } -bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName) + : Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath)) { - return outPath == other.outPath; +} + +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName) + : MissingRealisation{store, drvPathResolved, outputName} +{ + addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store)); } } // namespace nix @@ -74,24 +91,32 @@ using namespace nix; DrvOutput adl_serializer::from_json(const json & json) { - return DrvOutput::parse(getString(json)); + auto obj = getObject(json); + + return { + .drvPath = valueAt(obj, "drvPath"), + .outputName = getString(valueAt(obj, "outputName")), + }; } void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) { - json = drvOutput.to_string(); + json = { + {"drvPath", drvOutput.drvPath}, + {"outputName", drvOutput.outputName}, + }; } -UnkeyedRealisation adl_serializer::from_json(const json & json0) +UnkeyedRealisation adl_serializer::from_json(const json & json) { - auto json = getObject(json0); + auto obj = getObject(json); StringSet signatures; - if (auto signaturesOpt = optionalValueAt(json, "signatures")) - signatures = *signaturesOpt; + if (auto * signaturesJson = get(obj, "signatures")) + signatures = getStringSet(*signaturesJson); - return UnkeyedRealisation{ - .outPath = valueAt(json, "outPath"), + return { + .outPath = valueAt(obj, "outPath"), .signatures = signatures, }; } @@ -101,25 +126,25 @@ void adl_serializer::to_json(json & json, const UnkeyedReali json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - // back-compat - {"dependentRealisations", json::object()}, }; } -Realisation adl_serializer::from_json(const json & json0) +Realisation adl_serializer::from_json(const json & json) { - auto json = getObject(json0); + auto obj = getObject(json); - return Realisation{ - static_cast(json0), - valueAt(json, "id"), + return { + static_cast(valueAt(obj, "value")), + static_cast(valueAt(obj, "key")), }; } void adl_serializer::to_json(json & json, const Realisation & r) { - json = static_cast(r); - json["id"] = r.id; + json = { + {"key", r.id}, + {"value", static_cast(r)}, + }; } } // namespace nlohmann diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 949a51f18..6a8e48a6d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -507,7 +507,7 @@ void RemoteStore::queryRealisationUncached( try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 39) { warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -516,21 +516,12 @@ void RemoteStore::queryRealisationUncached( conn->to << id.to_string(); conn.processStderr(); - auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { - auto outPaths = WorkerProto::Serialise>::read(*this, *conn); - if (outPaths.empty()) - return nullptr; - return std::make_shared(UnkeyedRealisation{.outPath = *outPaths.begin()}); - } else { - auto realisations = WorkerProto::Serialise>::read(*this, *conn); - if (realisations.empty()) - return nullptr; - return std::make_shared(*realisations.begin()); - } - }(); - - callback(std::shared_ptr(real)); + callback([&]() -> std::shared_ptr { + auto realisation = WorkerProto::Serialise>::read(*this, *conn); + if (!realisation) + return nullptr; + return std::make_shared(*realisation); + }()); } catch (...) { return callback.rethrow(); } @@ -612,30 +603,19 @@ std::vector RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); - auto drv = evalStore->readDerivation(drvPath); - const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - printStorePath(drvPath), - output); - auto outputId = DrvOutput{*outputHash, output}; + auto outputId = DrvOutput{drvPath, output}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw MissingRealisation(outputId); - success.builtOutputs.emplace(output, Realisation{*realisation, outputId}); + throw MissingRealisation(*this, outputId); + success.builtOutputs.emplace(output, *realisation); } else { success.builtOutputs.emplace( output, - Realisation{ - UnkeyedRealisation{ - .outPath = outputPath, - }, - outputId, + UnkeyedRealisation{ + .outPath = outputPath, }); } } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index b4d696a53..99650870e 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -281,9 +281,18 @@ std::vector RestrictedStore::buildPathsWithResults( for (auto & result : results) { if (auto * successP = result.tryGetSuccess()) { - for (auto & [outputName, output] : successP->builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert(output); + if (auto * pathBuilt = std::get_if(&result.path)) { + // TODO ugly extra IO + auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath); + for (auto & [outputName, output] : successP->builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert( + {output, + { + .drvPath = drvPath, + .outputName = outputName, + }}); + } } } } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 51b575fcd..d5f8a9ea9 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -6,6 +6,7 @@ #include "nix/store/serve-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include @@ -24,10 +25,19 @@ BuildResult ServeProto::Serialise::read(const StoreDirConfig & stor if (GET_PROTOCOL_MINOR(conn.version) >= 3) conn.from >> status.timesBuilt >> failure.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { - auto builtOutputs = ServeProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (GET_PROTOCOL_MINOR(conn.version) >= 8) { + success.builtOutputs = ServeProto::Serialise>::read(store, conn); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + for (auto & [output, realisation] : ServeProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } if (BuildResult::Success::statusIs(rawStatus)) { @@ -51,15 +61,18 @@ void ServeProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - ServeProto::write(store, conn, builtOutputsFullKey); + + if (GET_PROTOCOL_MINOR(conn.version) >= 8) { + ServeProto::write(store, conn, builtOutputs); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + // We no longer support these types of realisations + ServeProto::write(store, conn, StringMap{}); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -144,4 +157,82 @@ void ServeProto::Serialise::write( } } +UnkeyedRealisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto outPath = ServeProto::Serialise::read(store, conn); + auto signatures = ServeProto::Serialise::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void ServeProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + ServeProto::write(store, conn, info.outPath); + ServeProto::write(store, conn, info.signatures); +} + +DrvOutput ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto drvPath = ServeProto::Serialise::read(store, conn); + auto outputName = ServeProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + ServeProto::write(store, conn, info.drvPath); + ServeProto::write(store, conn, info.outputName); +} + +Realisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = ServeProto::Serialise::read(store, conn); + auto unkeyed = ServeProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + ServeProto::write(store, conn, info.id); + ServeProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c70af9301..71642431f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -367,9 +367,8 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore return outputs; auto drv = evalStore.readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto & [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); + for (auto & [outputName, _] : drv.outputs) { + auto realisation = queryRealisation(DrvOutput{path, outputName}); if (realisation) { outputs.insert_or_assign(outputName, realisation->outPath); } else { @@ -389,7 +388,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw MissingRealisation(printStorePath(path), outName); + throw MissingRealisation(*this, path, outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index c2ef730dc..35b0063db 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1870,7 +1870,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { .outPath = newInfo.path, }, - DrvOutput{oldinfo->outputHash, outputName}, + DrvOutput{ + .drvPath = drvPath, + .outputName = outputName, + }, }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) { store.signRealisation(thisRealisation); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a17d2c028..c6719974b 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,6 +6,7 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include #include @@ -132,7 +133,7 @@ void WorkerProto::Serialise::write( throw Error( "trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MAJOR(conn.version) >> 8, GET_PROTOCOL_MINOR(conn.version)); }, [&](std::monostate) { @@ -174,14 +175,24 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.from >> res.timesBuilt >> failure.isNonDeterministic >> res.startTime >> res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { res.cpuUser = WorkerProto::Serialise>::read(store, conn); res.cpuSystem = WorkerProto::Serialise>::read(store, conn); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (GET_PROTOCOL_MINOR(conn.version) >= 39) { + success.builtOutputs = WorkerProto::Serialise>::read(store, conn); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + for (auto && [output, realisation] : WorkerProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } if (BuildResult::Success::statusIs(rawStatus)) { @@ -205,20 +216,24 @@ void WorkerProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuSystem); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputsFullKey); + + if (GET_PROTOCOL_MINOR(conn.version) >= 39) { + WorkerProto::write(store, conn, builtOutputs); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + // Don't support those types of realisations anymore. + WorkerProto::write(store, conn, StringMap{}); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -309,4 +324,113 @@ void WorkerProto::Serialise::write( } } +UnkeyedRealisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.39) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto outPath = WorkerProto::Serialise::read(store, conn); + auto signatures = WorkerProto::Serialise::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.39) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + WorkerProto::write(store, conn, info.outPath); + WorkerProto::write(store, conn, info.signatures); +} + +std::optional +WorkerProto::Serialise>::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + // Hack to improve compat + (void) WorkerProto::Serialise::read(store, conn); + return std::nullopt; + } else { + auto temp = readNum(conn.from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return WorkerProto::Serialise::read(store, conn); + default: + throw Error("Invalid optional build trace from remote"); + } + } +} + +void WorkerProto::Serialise>::write( + const StoreDirConfig & store, WriteConn conn, const std::optional & info) +{ + if (!info) { + conn.to << uint8_t{0}; + } else { + conn.to << uint8_t{1}; + WorkerProto::write(store, conn, *info); + } +} + +DrvOutput WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.29) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto drvPath = WorkerProto::Serialise::read(store, conn); + auto outputName = WorkerProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.29) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + WorkerProto::write(store, conn, info.drvPath); + WorkerProto::write(store, conn, info.outputName); +} + +Realisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = WorkerProto::Serialise::read(store, conn); + auto unkeyed = WorkerProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + WorkerProto::write(store, conn, info.id); + WorkerProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index ffb77ddf1..1e52daadc 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -346,13 +346,11 @@ static int main_build_remote(int argc, char ** argv) optResult = std::move(res[0]); } - auto outputHashes = staticOutputHashes(*store, drv); std::set missingRealisations; StorePathSet missingPaths; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { - auto thisOutputHash = outputHashes.at(outputName); - auto thisOutputId = DrvOutput{thisOutputHash, outputName}; + auto thisOutputId = DrvOutput{*drvPath, outputName}; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); assert(optResult); @@ -362,7 +360,7 @@ static int main_build_remote(int argc, char ** argv) auto i = success.builtOutputs.find(outputName); assert(i != success.builtOutputs.end()); auto & newRealisation = i->second; - missingRealisations.insert(newRealisation); + missingRealisations.insert({newRealisation, thisOutputId}); missingPaths.insert(newRealisation.outPath); } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d23dce10b..0f894b9de 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -279,16 +279,8 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore output.second = DerivationOutput::Deferred{}; drv.env[output.first] = ""; } - auto hashesModulo = hashDerivationModulo(*evalStore, drv, true); - for (auto & output : drv.outputs) { - Hash h = hashesModulo.hashes.at(output.first); - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = DerivationOutput::InputAddressed{ - .path = outPath, - }; - drv.env[output.first] = store->printStorePath(outPath); - } + resolveInputAddressed(*evalStore, drv); } auto shellDrvPath = writeDerivation(*evalStore, drv); diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 93e9f0f95..6d5dcb400 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -163,10 +163,13 @@ StoreWrapper::queryPathInfo(char * path, int base32) } SV * -StoreWrapper::queryRawRealisation(char * outputId) +StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) PPCODE: try { - auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput{ + .drvPath = THIS->store->parseStorePath(drvPath), + .outputName = outputName, + }); if (realisation) XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); else diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 2f6ebcef5..c4cbf2c96 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -49,14 +49,14 @@ fi clearStore nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore -rm -r "$REMOTE_STORE_DIR/realisations" +rm -r "$REMOTE_STORE_DIR/build-trace" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # There's no easy way to check whether a realisation is present on the local # store − short of manually querying the db, but the build environment doesn't # have the sqlite binary − so we instead push things again, and check that the # realisations have correctly been pushed to the remote store nix copy --to "$REMOTE_STORE" --file ../simple.nix -if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then +if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace")" ]]; then echo "Realisations not rebuilt" exit 1 fi @@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm "$REMOTE_STORE_DIR"/realisations/* +rm -r "$REMOTE_STORE_DIR"/build-trace/* buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 From c893454926bf1dc96848fa0e48d6625f11fafdb7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Oct 2025 13:49:54 -0400 Subject: [PATCH 12/12] Fixup issues with bytes --- src/libstore-tests/dummy-store.cc | 3 ++- src/libstore/dummy-store.cc | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index f53fc03b4..3ac3c2651 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -1,6 +1,7 @@ #include #include +#include "nix/util/bytes.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/globals.hh" @@ -105,7 +106,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { auto sc = make_ref(); sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ .executable = false, - .contents = "asdf", + .contents = to_owned(as_bytes("asdf")), }}; return sc; }(), diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 2a4c21b5f..b4b1d1f61 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,5 +1,6 @@ #include "nix/store/store-registration.hh" #include "nix/util/archive.hh" +#include "nix/util/bytes.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/util/json-utils.hh" @@ -160,7 +161,7 @@ struct DummyStoreImpl : DummyStore .method = ContentAddressMethod::Raw::Text, .hash = hashString( HashAlgorithm::SHA256, - std::get(accessor->root->raw).contents), + to_str(std::get(accessor->root->raw).contents)), }; callback(std::move(info)); return; @@ -428,7 +429,7 @@ ref adl_serializer>::from_json(const json & json) auto vref = make_ref(v2); res->buildTrace.insert_or_visit( { - Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + StorePath{k0}, {{k1, vref}}, }, [&](auto & kv) { kv.second.insert_or_assign(k1, vref); }); @@ -464,7 +465,7 @@ void adl_serializer>::to_json(json & json, const ref auto obj = json::object(); val->buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + auto & obj2 = obj[k.to_string()] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = *v2; });