From e145632aef02757b8949fc35ce7adfb04e353a71 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Dec 2025 01:49:58 -0500 Subject: [PATCH] Add unit test for double floating drv substitution This test will be updated to track progress on #11928 --- it shows the issue currently. --- .../issue-11928/store-after.json | 114 ++++++++++++ .../issue-11928/store-before.json | 52 ++++++ .../issue-11928/substituter.json | 70 +++++++ src/libstore-tests/worker-substitution.cc | 173 ++++++++++++++++++ 4 files changed, 409 insertions(+) create mode 100644 src/libstore-tests/data/worker-substitution/issue-11928/store-after.json create mode 100644 src/libstore-tests/data/worker-substitution/issue-11928/store-before.json create mode 100644 src/libstore-tests/data/worker-substitution/issue-11928/substituter.json diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json new file mode 100644 index 000000000..0d2cd439c --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -0,0 +1,114 @@ +{ + "buildTrace": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "out": { + "dependentRealisations": { + "sha256:82746e2bec1f6d7a913f380ee4cca205e6d7ad5d742b3bfeb6464212e3e6ee96!out": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out" + }, + "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", + "signatures": [] + } + }, + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": { + "contents": { + "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "narSize": 232, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { + "contents": { + "contents": "I am the dependency output", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "narSize": 144, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "dep-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json new file mode 100644 index 000000000..47a3b34be --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json @@ -0,0 +1,52 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "dep-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json new file mode 100644 index 000000000..c3530b8d5 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -0,0 +1,70 @@ +{ + "buildTrace": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "out": { + "dependentRealisations": { + "sha256:82746e2bec1f6d7a913f380ee4cca205e6d7ad5d742b3bfeb6464212e3e6ee96!out": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out" + }, + "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", + "signatures": [] + } + }, + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": { + "contents": { + "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "narSize": 232, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { + "contents": { + "contents": "I am the dependency output", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "narSize": 144, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 6a4ee46c8..016f6207b 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -273,4 +273,177 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) experimentalFeatureSettings.set("extra-experimental-features", ""); } +/** + * Test for issue #11928: substituting a CA derivation output should not + * require fetching the output of an input derivation when that output + * is not referenced. + */ +TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) +{ + // Enable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + + // Create the dependency CA floating derivation + Derivation depDrv; + depDrv.name = "dep-drv"; + depDrv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + + // Write the dependency derivation to the destination store + auto depDrvPath = writeDerivation(*dummyStore, depDrv); + + // Compute the hash modulo for the dependency derivation + auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); + ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); + auto depDrvHash = depHashModulo.hashes.at("out"); + + // Create the output store object for the dependency in the substituter + auto depOutputPath = substituter->addToStore( + "dep-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I am the dependency output", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Add the realisation for the dependency to the substituter + substituter->buildTrace.insert_or_assign( + depDrvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = depOutputPath, + }, + }, + }); + + // Create the root CA floating derivation that depends on depDrv + Derivation rootDrv; + rootDrv.name = "root-drv"; + rootDrv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + // Add the dependency derivation as an input + rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + + // Write the root derivation to the destination store + auto rootDrvPath = writeDerivation(*dummyStore, rootDrv); + + // Snapshot the destination store before + checkpointJson("issue-11928/store-before", dummyStore); + + // Compute the hash modulo for the root derivation + auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); + ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); + auto rootDrvHash = rootHashModulo.hashes.at("out"); + + // Create the output store object for the root derivation + // Note: it does NOT reference the dependency's output + auto rootOutputPath = substituter->addToStore( + "root-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = + "I am the root output. " + "I don't reference anything because the other derivation's output is just needed at build time.", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // The DrvOutputs for both derivations + DrvOutput depDrvOutput{depDrvHash, "out"}; + DrvOutput rootDrvOutput{rootDrvHash, "out"}; + + // Add the realisation for the root derivation to the substituter + // Include the dependency realisation in dependentRealisations + substituter->buildTrace.insert_or_assign( + rootDrvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = rootOutputPath, + .dependentRealisations = {{depDrvOutput, depOutputPath}}, + }, + }, + }); + + // Snapshot the substituter + // Note: it has realisations for both drvs, but only the root's output store object + checkpointJson("issue-11928/substituter", substituter); + + // The realisations should not exist in the destination store yet + ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput)); + ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituterAsStore = substituter; + worker.getSubstituters = [substituterAsStore]() -> std::list> { return {substituterAsStore}; }; + + // Create a derivation goal for the root derivation output + // The worker should substitute the output rather than building + auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false); + + // Run the worker + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after + checkpointJson("issue-11928/store-after", dummyStore); + + // The root output path should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath)); + + // The root realisation should now exist in the destination store + auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput); + ASSERT_TRUE(rootRealisation); + ASSERT_EQ(rootRealisation->outPath, rootOutputPath); + + // The dependency's REALISATION should have been fetched + auto depRealisation = dummyStore->queryRealisation(depDrvOutput); + ASSERT_TRUE(depRealisation); + ASSERT_EQ(depRealisation->outPath, depOutputPath); + + // TODO #11928: The dependency's OUTPUT should NOT be fetched (not referenced + // by root output). Once #11928 is fixed, change ASSERT_TRUE to ASSERT_FALSE. + ASSERT_TRUE(dummyStore->isValidPath(depOutputPath)); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); + + // Disable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", ""); +} + } // namespace nix