diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json new file mode 100644 index 000000000..7f0f62f87 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -0,0 +1,58 @@ +{ + "buildTrace": { + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": { + "contents": { + "contents": "I am the output of a CA derivation", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "narSize": 152, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "test-ca-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-before.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-before.json new file mode 100644 index 000000000..6d64d2327 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-before.json @@ -0,0 +1,27 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "test-ca-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json new file mode 100644 index 000000000..93f4fb22a --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -0,0 +1,39 @@ +{ + "buildTrace": { + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": { + "contents": { + "contents": "I am the output of a CA derivation", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "narSize": 152, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json new file mode 100644 index 000000000..0d2cd439c --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -0,0 +1,114 @@ +{ + "buildTrace": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "out": { + "dependentRealisations": { + "sha256:82746e2bec1f6d7a913f380ee4cca205e6d7ad5d742b3bfeb6464212e3e6ee96!out": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out" + }, + "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", + "signatures": [] + } + }, + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": { + "contents": { + "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "narSize": 232, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { + "contents": { + "contents": "I am the dependency output", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "narSize": 144, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "dep-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json new file mode 100644 index 000000000..47a3b34be --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json @@ -0,0 +1,52 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "dep-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json new file mode 100644 index 000000000..c3530b8d5 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -0,0 +1,70 @@ +{ + "buildTrace": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "out": { + "dependentRealisations": { + "sha256:82746e2bec1f6d7a913f380ee4cca205e6d7ad5d742b3bfeb6464212e3e6ee96!out": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out" + }, + "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", + "signatures": [] + } + }, + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": { + "contents": { + "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "narSize": 232, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { + "contents": { + "contents": "I am the dependency output", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "narSize": 144, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/single/substituter.json b/src/libstore-tests/data/worker-substitution/single/substituter.json new file mode 100644 index 000000000..f22d4c7df --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/single/substituter.json @@ -0,0 +1,31 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": { + "axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": { + "contents": { + "contents": "Hello, world!", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=", + "narSize": 128, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/with-dep/store.json b/src/libstore-tests/data/worker-substitution/with-dep/store.json new file mode 100644 index 000000000..3f2994dfb --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/with-dep/store.json @@ -0,0 +1,55 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": { + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": { + "contents": { + "contents": "I am a dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "narSize": 136, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { + "contents": { + "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "narSize": 184, + "references": [ + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency" + ], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json new file mode 100644 index 000000000..3f2994dfb --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json @@ -0,0 +1,55 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": { + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": { + "contents": { + "contents": "I am a dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "narSize": 136, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { + "contents": { + "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "narSize": 184, + "references": [ + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency" + ], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 58f624611..123bccb90 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -85,6 +85,7 @@ sources = files( 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol.cc', + 'worker-substitution.cc', 'write-derivation.cc', ) diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc new file mode 100644 index 000000000..016f6207b --- /dev/null +++ b/src/libstore-tests/worker-substitution.cc @@ -0,0 +1,449 @@ +#include +#include + +#include "nix/store/build/worker.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/globals.hh" +#include "nix/util/memory-source-accessor.hh" + +#include "nix/store/tests/libstore.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest> +{ + std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution"; + +protected: + ref dummyStore; + ref substituter; + + WorkerSubstitutionTest() + : LibStoreTest([] { + auto config = make_ref(DummyStoreConfig::Params{}); + config->readOnly = false; + return config->openDummyStore(); + }()) + , dummyStore(store.dynamic_pointer_cast()) + , substituter([] { + auto config = make_ref(DummyStoreConfig::Params{}); + config->readOnly = false; + config->isTrusted = true; + return config->openDummyStore(); + }()) + { + } + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + static void SetUpTestSuite() + { + initLibStore(false); + } +}; + +TEST_F(WorkerSubstitutionTest, singleStoreObject) +{ + // Add a store path to the substituter + auto pathInSubstituter = substituter->addToStore( + "hello", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "Hello, world!", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Snapshot the substituter (has one store object) + checkpointJson("single/substituter", substituter); + + // Snapshot the destination store before (should be empty) + checkpointJson("../dummy-store/empty", dummyStore); + + // The path should not exist in the destination store yet + ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituerAsStore = substituter; + worker.getSubstituters = [substituerAsStore]() -> std::list> { return {substituerAsStore}; }; + + // Create a substitution goal for the path + auto goal = worker.makePathSubstitutionGoal(pathInSubstituter); + + // Run the worker with -j0 semantics (no local builds, only substitution) + // The worker.run() takes a set of goals + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after (should match the substituter) + checkpointJson("single/substituter", dummyStore); + + // The path should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter)); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); +} + +TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject) +{ + // First, add a dependency store path to the substituter + auto dependencyPath = substituter->addToStore( + "dependency", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I am a dependency", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Now add a store path that references the dependency + auto mainPath = substituter->addToStore( + "main", + SourcePath{ + [&] { + auto sc = make_ref(); + // Include a reference to the dependency path in the contents + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I depend on " + substituter->printStorePath(dependencyPath), + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256, + StorePathSet{dependencyPath}); + + // Snapshot the substituter (has two store objects) + checkpointJson("with-dep/substituter", substituter); + + // Snapshot the destination store before (should be empty) + checkpointJson("../dummy-store/empty", dummyStore); + + // Neither path should exist in the destination store yet + ASSERT_FALSE(dummyStore->isValidPath(dependencyPath)); + ASSERT_FALSE(dummyStore->isValidPath(mainPath)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituterAsStore = substituter; + worker.getSubstituters = [substituterAsStore]() -> std::list> { return {substituterAsStore}; }; + + // Create a substitution goal for the main path only + // The worker should automatically substitute the dependency as well + auto goal = worker.makePathSubstitutionGoal(mainPath); + + // Run the worker + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after (should match the substituter) + checkpointJson("with-dep/substituter", dummyStore); + + // Both paths should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(dependencyPath)); + ASSERT_TRUE(dummyStore->isValidPath(mainPath)); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); +} + +TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) +{ + // Enable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + + // Create a CA floating output derivation + Derivation drv; + drv.name = "test-ca-drv"; + drv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + + // Write the derivation to the destination store + auto drvPath = writeDerivation(*dummyStore, drv); + + // Snapshot the destination store before + checkpointJson("ca-drv/store-before", dummyStore); + + // Compute the hash modulo of the derivation + // For CA floating derivations, the kind is Deferred since outputs aren't known until build + auto hashModulo = hashDerivationModulo(*dummyStore, drv, true); + ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred); + auto drvHash = hashModulo.hashes.at("out"); + + // Create the output store object + auto outputPath = substituter->addToStore( + "test-ca-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I am the output of a CA derivation", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Add the realisation (build trace) to the substituter + substituter->buildTrace.insert_or_assign( + drvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = outputPath, + }, + }, + }); + + // Snapshot the substituter + checkpointJson("ca-drv/substituter", substituter); + + // The realisation should not exist in the destination store yet + DrvOutput drvOutput{drvHash, "out"}; + ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituterAsStore = substituter; + worker.getSubstituters = [substituterAsStore]() -> std::list> { return {substituterAsStore}; }; + + // Create a derivation goal for the CA derivation output + // The worker should substitute the output rather than building + auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true); + + // Run the worker + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after + checkpointJson("ca-drv/store-after", dummyStore); + + // The output path should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(outputPath)); + + // The realisation should now exist in the destination store + auto realisation = dummyStore->queryRealisation(drvOutput); + ASSERT_TRUE(realisation); + ASSERT_EQ(realisation->outPath, outputPath); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); + + // Disable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", ""); +} + +/** + * Test for issue #11928: substituting a CA derivation output should not + * require fetching the output of an input derivation when that output + * is not referenced. + */ +TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) +{ + // Enable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + + // Create the dependency CA floating derivation + Derivation depDrv; + depDrv.name = "dep-drv"; + depDrv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + + // Write the dependency derivation to the destination store + auto depDrvPath = writeDerivation(*dummyStore, depDrv); + + // Compute the hash modulo for the dependency derivation + auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); + ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); + auto depDrvHash = depHashModulo.hashes.at("out"); + + // Create the output store object for the dependency in the substituter + auto depOutputPath = substituter->addToStore( + "dep-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I am the dependency output", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Add the realisation for the dependency to the substituter + substituter->buildTrace.insert_or_assign( + depDrvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = depOutputPath, + }, + }, + }); + + // Create the root CA floating derivation that depends on depDrv + Derivation rootDrv; + rootDrv.name = "root-drv"; + rootDrv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + // Add the dependency derivation as an input + rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + + // Write the root derivation to the destination store + auto rootDrvPath = writeDerivation(*dummyStore, rootDrv); + + // Snapshot the destination store before + checkpointJson("issue-11928/store-before", dummyStore); + + // Compute the hash modulo for the root derivation + auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); + ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); + auto rootDrvHash = rootHashModulo.hashes.at("out"); + + // Create the output store object for the root derivation + // Note: it does NOT reference the dependency's output + auto rootOutputPath = substituter->addToStore( + "root-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = + "I am the root output. " + "I don't reference anything because the other derivation's output is just needed at build time.", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // The DrvOutputs for both derivations + DrvOutput depDrvOutput{depDrvHash, "out"}; + DrvOutput rootDrvOutput{rootDrvHash, "out"}; + + // Add the realisation for the root derivation to the substituter + // Include the dependency realisation in dependentRealisations + substituter->buildTrace.insert_or_assign( + rootDrvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = rootOutputPath, + .dependentRealisations = {{depDrvOutput, depOutputPath}}, + }, + }, + }); + + // Snapshot the substituter + // Note: it has realisations for both drvs, but only the root's output store object + checkpointJson("issue-11928/substituter", substituter); + + // The realisations should not exist in the destination store yet + ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput)); + ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituterAsStore = substituter; + worker.getSubstituters = [substituterAsStore]() -> std::list> { return {substituterAsStore}; }; + + // Create a derivation goal for the root derivation output + // The worker should substitute the output rather than building + auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false); + + // Run the worker + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after + checkpointJson("issue-11928/store-after", dummyStore); + + // The root output path should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath)); + + // The root realisation should now exist in the destination store + auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput); + ASSERT_TRUE(rootRealisation); + ASSERT_EQ(rootRealisation->outPath, rootOutputPath); + + // The dependency's REALISATION should have been fetched + auto depRealisation = dummyStore->queryRealisation(depDrvOutput); + ASSERT_TRUE(depRealisation); + ASSERT_EQ(depRealisation->outPath, depOutputPath); + + // TODO #11928: The dependency's OUTPUT should NOT be fetched (not referenced + // by root output). Once #11928 is fixed, change ASSERT_TRUE to ASSERT_FALSE. + ASSERT_TRUE(dummyStore->isValidPath(depOutputPath)); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); + + // Disable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", ""); +} + +} // namespace nix diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 03ebb38bb..171aa900a 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -3,8 +3,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/util/callback.hh" -#include "nix/store/store-open.hh" -#include "nix/store/globals.hh" namespace nix { @@ -25,7 +23,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() co_return amDone(ecSuccess); } - auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + auto subs = worker.getSubstituters(); bool substituterFailed = false; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 51dc87e8d..b1273835c 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,5 +1,4 @@ #include "nix/store/build/worker.hh" -#include "nix/store/store-open.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/nar-info.hh" #include "nix/util/finally.hh" @@ -60,7 +59,7 @@ Goal::Co PathSubstitutionGoal::init() throw Error( "cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + auto subs = worker.getSubstituters(); bool substituterFailed = false; std::optional lastStoresException = std::nullopt; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 754f4b1cc..593319d03 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,5 +1,6 @@ #include "nix/store/local-store.hh" #include "nix/store/machines.hh" +#include "nix/store/store-open.hh" #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/drv-output-substitution-goal.hh" @@ -21,6 +22,7 @@ Worker::Worker(Store & store, Store & evalStore) , actSubstitutions(*logger, actCopyPaths) , store(store) , evalStore(evalStore) + , getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list>{}; }} { nrLocalBuilds = 0; nrSubstitutions = 0; diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 173f7b222..65408429c 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -8,6 +8,7 @@ #include "nix/store/realisation.hh" #include "nix/util/muxable-pipe.hh" +#include #include #include @@ -171,6 +172,14 @@ public: Store & store; Store & evalStore; + /** + * Function to get the substituters to use for path substitution. + * + * Defaults to `getDefaultSubstituters`. This allows tests to + * inject custom substituters. + */ + std::function>()> getSubstituters; + #ifndef _WIN32 // TODO Enable building on Windows std::unique_ptr hook; #endif diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index 6db32c4b6..744880f38 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -83,6 +83,31 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go } } +/** + * Specialization for when we need to do "JSON -> `ref`" in one + * direction, but "`const T &` -> JSON" in the other direction. + */ +template +void checkpointJson(CharacterizationTest & test, PathView testStem, const ref & got) +{ + using namespace nlohmann; + + auto file = test.goldenMaster(Path{testStem} + ".json"); + + json gotJson = static_cast(*got); + + if (testAccept()) { + std::filesystem::create_directories(file.parent_path()); + writeFile(file, gotJson.dump(2) + "\n"); + ADD_FAILURE() << "Updating golden master " << file; + } else { + json expectedJson = json::parse(readFile(file)); + ASSERT_EQ(gotJson, expectedJson); + ref expected = adl_serializer>::from_json(expectedJson); + ASSERT_EQ(*got, *expected); + } +} + /** * Mixin class for writing characterization tests for `nlohmann::json` * conversions for a given type.