nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-09 12:06:01 +01:00
Code Activity

Merge branch 'build-trace-rework' into json-no-store-dir

Browse source
This commit is contained in:
John Ericson 2025-10-23 13:44:24 -04:00
commit 04edfa34fe
63 changed files with 975 additions and 992 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/libcmd/built-path.cc
View file

@ -108,20 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
overloaded{ overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) { [&](const BuiltPath::Built & p) {
auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
for (auto & [outputName, outputPath] : p.outputs) { for (auto & [outputName, outputPath] : p.outputs) {
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName); DrvOutput key{
if (!drvOutput) .drvPath = p.drvPath->outPath(),
throw Error( .outputName = outputName,
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", };
store.printStorePath(p.drvPath->outPath()),
outputName);
DrvOutput key{*drvOutput, outputName};
auto thisRealisation = store.queryRealisation(key); auto thisRealisation = store.queryRealisation(key);
assert(thisRealisation); // We’ve built it, so we must // We’ve built it, so we must have the realisation.
// have the realisation assert(thisRealisation);
res.insert(Realisation{*thisRealisation, std::move(key)}); res.insert(Realisation{*thisRealisation, key});
} else { } else {
res.insert(outputPath); res.insert(outputPath);
} }

23
src/libexpr/primops.cc
View file

@ -1718,28 +1718,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{}); drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{});
} }
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); resolveInputAddressed(*state.store, drv);
switch (hashModulo.kind) {
case DrvHash::Kind::Regular:
for (auto & i : outputs) {
auto h = get(hashModulo.hashes, i);
if (!h)
state.error<AssertionError>("derivation produced no hash for output '%s'", i).atPos(v).debugThrow();
auto outPath = state.store->makeOutputPath(i, *h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
i,
DerivationOutput::InputAddressed{
.path = std::move(outPath),
});
}
break;
;
case DrvHash::Kind::Deferred:
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{});
}
}
} }
/* Write the resulting term into the Nix store directory. */ /* Write the resulting term into the Nix store directory. */

10
src/libstore-test-support/include/nix/store/tests/protocol.hh
View file

@ -69,14 +69,20 @@ public:
} }
}; };
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ #define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_read) \ TEST_F(FIXTURE, NAME##_read) \
{ \ { \
readProtoTest(STEM, VERSION, VALUE); \ readProtoTest(STEM, VERSION, VALUE); \
} \ }
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_write) \ TEST_F(FIXTURE, NAME##_write) \
{ \ { \
writeProtoTest(STEM, VERSION, VALUE); \ writeProtoTest(STEM, VERSION, VALUE); \
} }
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
} // namespace nix } // namespace nix

60
src/libstore-tests/common-protocol.cc
View file

@ -46,16 +46,22 @@ public:
} }
}; };
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ #define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
TEST_F(CommonProtoTest, NAME##_read) \ TEST_F(CommonProtoTest, NAME##_read) \
{ \ { \
readProtoTest(STEM, VALUE); \ readProtoTest(STEM, VALUE); \
} \ }
#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
TEST_F(CommonProtoTest, NAME##_write) \ TEST_F(CommonProtoTest, NAME##_write) \
{ \ { \
writeProtoTest(STEM, VALUE); \ writeProtoTest(STEM, VALUE); \
} }
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE)
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
string, string,
"string", "string",
@ -93,56 +99,6 @@ CHARACTERIZATION_TEST(
}, },
})) }))
CHARACTERIZATION_TEST(
drvOutput,
"drv-output",
(std::tuple<DrvOutput, DrvOutput>{
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
}))
CHARACTERIZATION_TEST(
realisation,
"realisation",
(std::tuple<Realisation, Realisation>{
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
},
DrvOutput{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
},
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
.dependentRealisations =
{
{
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
},
}))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
vector, vector,
"vector", "vector",

BIN
src/libstore-tests/data/common-protocol/realisation-with-deps.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/common-protocol/realisation.bin
View file

Binary file not shown.

10
src/libstore-tests/data/realisation/simple.json
View file

@ -1,6 +1,10 @@
{ {
"dependentRealisations": {}, "key": {
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", "outputName": "foo"
},
"value": {
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [] "signatures": []
}
} }

10
src/libstore-tests/data/realisation/with-signature.json
View file

@ -1,8 +1,12 @@
{ {
"dependentRealisations": {}, "key": {
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", "outputName": "foo"
},
"value": {
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [ "signatures": [
"asdfasdfasdf" "asdfasdfasdf"
] ]
}
} }

BIN
src/libstore-tests/data/serve-protocol/build-result-2.8.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/serve-protocol/drv-output-2.8.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/serve-protocol/realisation-2.8.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/serve-protocol/realisation-with-deps.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/serve-protocol/realisation.bin
View file

Binary file not shown.

BIN
src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/worker-protocol/build-result-1.39.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/worker-protocol/drv-output-1.39.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/worker-protocol/realisation-1.39.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/worker-protocol/realisation-with-deps.bin Normal file
View file

Binary file not shown.

BIN
src/libstore-tests/data/worker-protocol/realisation.bin
View file

Binary file not shown.

BIN
src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin Normal file
View file

Binary file not shown.

11
src/libstore-tests/dummy-store.cc
View file

@ -37,20 +37,19 @@ TEST(DummyStore, realisation_read)
return cfg->openDummyStore(); return cfg->openDummyStore();
}(); }();
auto drvHash = Hash::parseExplicitFormatUnprefixed( StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"};
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16);
auto outputName = "foo"; auto outputName = "foo";
EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr);
UnkeyedRealisation value{ UnkeyedRealisation value{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
}; };
store->buildTrace.insert({drvHash, {{outputName, make_ref<UnkeyedRealisation>(value)}}}); store->buildTrace.insert({drvPath, {{outputName, make_ref<UnkeyedRealisation>(value)}}});
auto value2 = store->queryRealisation({drvHash, outputName}); auto value2 = store->queryRealisation({drvPath, outputName});
ASSERT_TRUE(value2); ASSERT_TRUE(value2);
EXPECT_EQ(*value2, value); EXPECT_EQ(*value2, value);

32
src/libstore-tests/realisation.cc
View file

@ -44,19 +44,13 @@ TEST_P(RealisationJsonTest, to_json)
writeJsonTest(name, value); writeJsonTest(name, value);
} }
INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(RealisationJSON, RealisationJsonTest, ([] {
RealisationJSON,
RealisationJsonTest,
([] {
Realisation simple{ Realisation simple{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
}, },
{ {
.drvHash = Hash::parseExplicitFormatUnprefixed( .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"},
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
HashAlgorithm::SHA256,
HashFormat::Base16),
.outputName = "foo", .outputName = "foo",
}, },
}; };
@ -72,26 +66,8 @@ INSTANTIATE_TEST_SUITE_P(
// FIXME actually sign properly // FIXME actually sign properly
r.signatures = {"asdfasdfasdf"}; r.signatures = {"asdfasdfasdf"};
return r; return r;
}()},
std::pair{
"with-dependent-realisations",
[&] {
auto r = simple;
r.dependentRealisations = {{
{
.drvHash = Hash::parseExplicitFormatUnprefixed(
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
HashAlgorithm::SHA256,
HashFormat::Base16),
.outputName = "foo",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
}};
return r;
}(), }(),
}); });
} }()));
()));
} // namespace nix } // namespace nix

121
src/libstore-tests/serve-protocol.cc
View file

@ -72,16 +72,16 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest, ServeProtoTest,
drvOutput, drvOutput_2_8,
"drv-output", "drv-output-2.8",
defaultVersion, 2 << 8 | 8,
(std::tuple<DrvOutput, DrvOutput>{ (std::tuple<DrvOutput, DrvOutput>{
{ {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
.outputName = "baz", .outputName = "baz",
}, },
DrvOutput{ DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
.outputName = "quux", .outputName = "quux",
}, },
})) }))
@ -90,40 +90,28 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest, ServeProtoTest,
realisation, unkeyedRealisation_2_8,
"realisation", "unkeyed-realisation-2.8",
defaultVersion, 2 << 8 | 8,
(std::tuple<Realisation, Realisation>{ (UnkeyedRealisation{
Realisation{ .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
{ .signatures = {"asdf", "qwer"},
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
realisation_2_8,
"realisation-2.8",
2 << 8 | 8,
(Realisation{
UnkeyedRealisation{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"}, .signatures = {"asdf", "qwer"},
}, },
{ {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
.outputName = "baz", .outputName = "baz",
}, },
},
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
.dependentRealisations =
{
{
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
},
})) }))
VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2.2", 2 << 8 | 2, ({ VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2.2", 2 << 8 | 2, ({
@ -172,7 +160,10 @@ VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2
t; t;
})) }))
VERSIONED_CHARACTERIZATION_TEST( /* We now do a lossy read which does not allow us to faithfully right
back, since we changed the data type. We still however want to test
that this read works, and so for that we have a one-way test. */
VERSIONED_READ_CHARACTERIZATION_TEST(
ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({ ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{ std::tuple<BuildResult, BuildResult, BuildResult> t{
@ -197,29 +188,67 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
{ {
"foo", "foo",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
}, },
DrvOutput{
.drvHash =
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
},
}, },
{ {
"bar", "bar",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
}, },
DrvOutput{
.drvHash =
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
}, },
}, },
}},
.timesBuilt = 1,
.startTime = 30,
.stopTime = 50,
#if 0
// These fields are not yet serialized.
// FIXME Include in next version of protocol or document
// why they are skipped.
.cpuUser = std::chrono::milliseconds(500s),
.cpuSystem = std::chrono::milliseconds(604s),
#endif
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest, buildResult_2_8, "build-result-2.8", 2 << 8 | 8, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.builtOutputs =
{
{
"foo",
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
{
"bar",
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
},
}, },
}, },
}}, }},

148
src/libstore-tests/worker-protocol.cc
View file

@ -128,55 +128,43 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest, WorkerProtoTest,
drvOutput, drvOutput,
"drv-output", "drv-output-1.39",
defaultVersion, 1 << 8 | 39,
(std::tuple<DrvOutput, DrvOutput>{ (std::tuple<DrvOutput, DrvOutput>{
{ {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
.outputName = "baz", .outputName = "baz",
}, },
DrvOutput{ DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
.outputName = "quux", .outputName = "quux",
}, },
})) }))
VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest, WorkerProtoTest,
realisation, unkeyedRealisation_1_39,
"realisation", "unkeyed-realisation-1.39",
defaultVersion, 1 << 8 | 39,
(std::tuple<Realisation, Realisation>{ (UnkeyedRealisation{
Realisation{ .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
{ .signatures = {"asdf", "qwer"},
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
realisation_1_39,
"realisation-1.39",
1 << 8 | 39,
(Realisation{
UnkeyedRealisation{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"}, .signatures = {"asdf", "qwer"},
}, },
DrvOutput{ {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
.outputName = "baz", .outputName = "baz",
}, },
},
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
.dependentRealisations =
{
{
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
},
DrvOutput{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
},
})) }))
VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result-1.27", 1 << 8 | 27, ({ VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result-1.27", 1 << 8 | 27, ({
@ -197,7 +185,10 @@ VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result
t; t;
})) }))
VERSIONED_CHARACTERIZATION_TEST( /* We now do a lossy read which does not allow us to faithfully right
back, since we changed the data type. We still however want to test
that this read works, and so for that we have a one-way test. */
VERSIONED_READ_CHARACTERIZATION_TEST(
WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({ WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{ std::tuple<BuildResult, BuildResult, BuildResult> t{
@ -215,27 +206,15 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
{ {
"foo", "foo",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
}, },
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
},
}, },
{ {
"bar", "bar",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
}, },
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
},
}, },
}, },
}}}, }}},
@ -243,7 +222,8 @@ VERSIONED_CHARACTERIZATION_TEST(
t; t;
})) }))
VERSIONED_CHARACTERIZATION_TEST( // See above note
VERSIONED_READ_CHARACTERIZATION_TEST(
WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({ WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{ std::tuple<BuildResult, BuildResult, BuildResult> t{
@ -268,29 +248,15 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
{ {
"foo", "foo",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
}, },
DrvOutput{
.drvHash =
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
},
}, },
{ {
"bar", "bar",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
}, },
DrvOutput{
.drvHash =
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
},
}, },
}, },
}}, }},
@ -302,7 +268,8 @@ VERSIONED_CHARACTERIZATION_TEST(
t; t;
})) }))
VERSIONED_CHARACTERIZATION_TEST( // See above note
VERSIONED_READ_CHARACTERIZATION_TEST(
WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({ WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{ std::tuple<BuildResult, BuildResult, BuildResult> t{
@ -327,29 +294,62 @@ VERSIONED_CHARACTERIZATION_TEST(
{ {
{ {
"foo", "foo",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
}, },
DrvOutput{
.drvHash =
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
},
}, },
{ {
"bar", "bar",
{
{ {
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
}, },
DrvOutput{
.drvHash =
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
}, },
}, },
}},
.timesBuilt = 1,
.startTime = 30,
.stopTime = 50,
.cpuUser = std::chrono::microseconds(500s),
.cpuSystem = std::chrono::microseconds(604s),
},
};
t;
}))
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest, buildResult_1_39, "build-result-1.39", 1 << 8 | 39, ({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t{
BuildResult{.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
}}},
BuildResult{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = true,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.builtOutputs =
{
{
"foo",
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
{
"bar",
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
},
}, },
}, },
}}, }},

12
src/libstore/binary-cache-store.cc
View file

@ -504,7 +504,7 @@ StorePath BinaryCacheStore::addToStore(
std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id)
{ {
return realisationsPrefix + "/" + id.to_string() + ".doi"; return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi";
} }
void BinaryCacheStore::queryRealisationUncached( void BinaryCacheStore::queryRealisationUncached(
@ -525,7 +525,10 @@ void BinaryCacheStore::queryRealisationUncached(
realisation = std::make_shared<const UnkeyedRealisation>(nlohmann::json::parse(*data)); realisation = std::make_shared<const UnkeyedRealisation>(nlohmann::json::parse(*data));
} catch (Error & e) { } catch (Error & e) {
e.addTrace( e.addTrace(
{}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); {},
"while parsing file '%s' as a build trace value for key '%s'",
outputInfoFilePath,
id.to_string());
throw; throw;
} }
return (*callbackPtr)(std::move(realisation)); return (*callbackPtr)(std::move(realisation));
@ -541,7 +544,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
{ {
if (diskCache) if (diskCache)
diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info);
upsertFile(makeRealisationPath(info.id), static_cast<nlohmann::json>(info).dump(), "application/json"); upsertFile(
makeRealisationPath(info.id),
static_cast<nlohmann::json>(static_cast<const UnkeyedRealisation &>(info)).dump(),
"application/json");
} }
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)

7
src/libstore/build/derivation-building-goal.cc
View file

@ -212,9 +212,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
given this information by the downstream goal, that cannot happen given this information by the downstream goal, that cannot happen
anymore if the downstream goal only cares about one output, but anymore if the downstream goal only cares about one output, but
we care about all outputs. */ we care about all outputs. */
auto outputHashes = staticOutputHashes(worker.evalStore, *drv); for (auto & [outputName, _] : drv->outputs) {
for (auto & [outputName, outputHash] : outputHashes) { InitialOutput v;
InitialOutput v{.outputHash = outputHash};
/* TODO we might want to also allow randomizing the paths /* TODO we might want to also allow randomizing the paths
for regular CA derivations, e.g. for sake of checking for regular CA derivations, e.g. for sake of checking
@ -1096,7 +1095,7 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
: PathStatus::Corrupt, : PathStatus::Corrupt,
}; };
} }
auto drvOutput = DrvOutput{info.outputHash, i.first}; auto drvOutput = DrvOutput{drvPath, i.first};
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
if (auto real = worker.store.queryRealisation(drvOutput)) { if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = { info.known = {

52
src/libstore/build/derivation-goal.cc
View file

@ -36,12 +36,6 @@ DerivationGoal::DerivationGoal(
, drvPath(drvPath) , drvPath(drvPath)
, wantedOutput(wantedOutput) , wantedOutput(wantedOutput)
, drv{std::make_unique<Derivation>(drv)} , drv{std::make_unique<Derivation>(drv)}
, outputHash{[&] {
auto outputHashes = staticOutputHashes(worker.evalStore, drv);
if (auto * mOutputHash = get(outputHashes, wantedOutput))
return *mOutputHash;
throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput);
}()}
, buildMode(buildMode) , buildMode(buildMode)
{ {
@ -100,7 +94,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
them. */ them. */
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) { if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (!checkResult) if (!checkResult)
waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput}))); waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{drvPath, wantedOutput})));
else { else {
auto * cap = getDerivationCA(*drv); auto * cap = getDerivationCA(*drv);
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal( waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
@ -167,12 +161,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
// No `std::visit` for coroutines yet // No `std::visit` for coroutines yet
if (auto * successP = resolvedResult.tryGetSuccess()) { if (auto * successP = resolvedResult.tryGetSuccess()) {
auto & success = *successP; auto & success = *successP;
auto outputHashes = staticOutputHashes(worker.evalStore, *drv); if (!drv->outputs.contains(wantedOutput))
auto resolvedHashes = staticOutputHashes(worker.store, drvResolved);
auto outputHash = get(outputHashes, wantedOutput);
auto resolvedHash = get(resolvedHashes, wantedOutput);
if ((!outputHash) || (!resolvedHash))
throw Error( throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)",
worker.store.printStorePath(drvPath), worker.store.printStorePath(drvPath),
@ -181,7 +170,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
auto realisation = [&] { auto realisation = [&] {
auto take1 = get(success.builtOutputs, wantedOutput); auto take1 = get(success.builtOutputs, wantedOutput);
if (take1) if (take1)
return static_cast<UnkeyedRealisation>(*take1); return *take1;
/* The above `get` should work. But stateful tracking of /* The above `get` should work. But stateful tracking of
outputs in resolvedResult, this can get out of sync with the outputs in resolvedResult, this can get out of sync with the
@ -189,7 +178,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
check the store directly if it fails. */ check the store directly if it fails. */
auto take2 = worker.evalStore.queryRealisation( auto take2 = worker.evalStore.queryRealisation(
DrvOutput{ DrvOutput{
.drvHash = *resolvedHash, .drvPath = pathResolved,
.outputName = wantedOutput, .outputName = wantedOutput,
}); });
if (take2) if (take2)
@ -205,15 +194,10 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
Realisation newRealisation{ Realisation newRealisation{
realisation, realisation,
{ {
.drvHash = *outputHash, .drvPath = drvPath,
.outputName = wantedOutput, .outputName = wantedOutput,
}}; }};
newRealisation.signatures.clear(); newRealisation.signatures.clear();
if (!drv->type().isFixed()) {
auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
newRealisation.dependentRealisations =
drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
}
worker.store.signRealisation(newRealisation); worker.store.signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
} }
@ -256,16 +240,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
/* In checking mode, the builder will not register any outputs. /* In checking mode, the builder will not register any outputs.
So we want to make sure the ones that we wanted to check are So we want to make sure the ones that we wanted to check are
properly there. */ properly there. */
success.builtOutputs = {{ success.builtOutputs = {{wantedOutput, assertPathValidity()}};
wantedOutput,
{
assertPathValidity(),
{
.drvHash = outputHash,
.outputName = wantedOutput,
},
},
}};
} else { } else {
/* Otherwise the builder will give us info for out output, but /* Otherwise the builder will give us info for out output, but
also for other outputs. Filter down to just our output so as also for other outputs. Filter down to just our output so as
@ -374,7 +349,7 @@ std::optional<std::pair<UnkeyedRealisation, PathStatus>> DerivationGoal::checkPa
if (drv->type().isImpure()) if (drv->type().isImpure())
return std::nullopt; return std::nullopt;
auto drvOutput = DrvOutput{outputHash, wantedOutput}; auto drvOutput = DrvOutput{drvPath, wantedOutput};
std::optional<UnkeyedRealisation> mRealisation; std::optional<UnkeyedRealisation> mRealisation;
@ -414,7 +389,7 @@ std::optional<std::pair<UnkeyedRealisation, PathStatus>> DerivationGoal::checkPa
Realisation{ Realisation{
*mRealisation, *mRealisation,
{ {
.drvHash = outputHash, .drvPath = drvPath,
.outputName = wantedOutput, .outputName = wantedOutput,
}, },
}); });
@ -437,16 +412,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
{ {
buildResult.inner = BuildResult::Success{ buildResult.inner = BuildResult::Success{
.status = status, .status = status,
.builtOutputs = {{ .builtOutputs = {{wantedOutput, std::move(builtOutput)}},
wantedOutput,
{
std::move(builtOutput),
DrvOutput{
.drvHash = outputHash,
.outputName = wantedOutput,
},
},
}},
}; };
mcExpectedBuilds.reset(); mcExpectedBuilds.reset();

31
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -12,7 +12,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke
: Goal(worker, init()) : Goal(worker, init())
, id(id) , id(id)
{ {
name = fmt("substitution of '%s'", id.to_string()); name = fmt("substitution of '%s'", id.render(worker.store));
trace("created"); trace("created");
} }
@ -86,32 +86,8 @@ Goal::Co DrvOutputSubstitutionGoal::init()
if (!outputInfo) if (!outputInfo)
continue; continue;
bool failed = false;
Goals waitees; Goals waitees;
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
if (depId != id) {
if (auto localOutputInfo = worker.store.queryRealisation(depId);
localOutputInfo && localOutputInfo->outPath != depPath) {
warn(
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
"Local: %s\n"
"Remote: %s",
sub->config.getHumanReadableURI(),
depId.to_string(),
worker.store.printStorePath(localOutputInfo->outPath),
worker.store.printStorePath(depPath));
failed = true;
break;
}
waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
}
}
if (failed)
continue;
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
co_await await(std::move(waitees)); co_await await(std::move(waitees));
@ -131,7 +107,8 @@ Goal::Co DrvOutputSubstitutionGoal::init()
/* None left. Terminate this goal and let someone else deal /* None left. Terminate this goal and let someone else deal
with it. */ with it. */
debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); debug(
"derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store));
if (substituterFailed) { if (substituterFailed) {
worker.failedSubstitutions++; worker.failedSubstitutions++;
@ -146,7 +123,7 @@ Goal::Co DrvOutputSubstitutionGoal::init()
std::string DrvOutputSubstitutionGoal::key() std::string DrvOutputSubstitutionGoal::key()
{ {
return "a$" + std::string(id.to_string()); return "a$" + std::string(id.render(worker.store));
} }
void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)

27
src/libstore/ca-specific-schema.sql
View file

@ -12,30 +12,3 @@ create table if not exists Realisations (
); );
create index if not exists IndexRealisations on Realisations(drvPath, outputName); create index if not exists IndexRealisations on Realisations(drvPath, outputName);
-- We can end-up in a weird edge-case where a path depends on itself because
-- it’s an output of a CA derivation, that happens to be the same as one of its
-- dependencies.
-- In that case we have a dependency loop (path -> realisation1 -> realisation2
-- -> path) that we need to break by removing the dependencies between the
-- realisations
create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths
begin
delete from RealisationsRefs where realisationReference in (
select id from Realisations where outputPath = old.id
);
end;
create table if not exists RealisationsRefs (
referrer integer not null,
realisationReference integer,
foreign key (referrer) references Realisations(id) on delete cascade,
foreign key (realisationReference) references Realisations(id) on delete restrict
);
-- used by deletion trigger
create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference);
-- used by QueryRealisationReferences
create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
-- used by cascade deletion when ValidPaths is deleted
create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);

28
src/libstore/common-protocol.cc
View file

@ -46,34 +46,6 @@ void CommonProto::Serialise<ContentAddress>::write(
conn.to << renderContentAddress(ca); conn.to << renderContentAddress(ca);
} }
Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
std::string rawInput = readString(conn.from);
try {
return nlohmann::json::parse(rawInput);
} catch (Error & e) {
e.addTrace({}, "while parsing a realisation object in the remote protocol");
throw;
}
}
void CommonProto::Serialise<Realisation>::write(
const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
{
conn.to << static_cast<nlohmann::json>(realisation).dump();
}
DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{
return DrvOutput::parse(readString(conn.from));
}
void CommonProto::Serialise<DrvOutput>::write(
const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
{
conn.to << drvOutput.to_string();
}
std::optional<StorePath> std::optional<StorePath>
CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
{ {

22
src/libstore/daemon.cc
View file

@ -963,33 +963,31 @@ static void performOp(
case WorkerProto::Op::RegisterDrvOutput: { case WorkerProto::Op::RegisterDrvOutput: {
logger->startWork(); logger->startWork();
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { // TODO move to WorkerProto::Serialise<DrvOutput> and friends
auto outputId = DrvOutput::parse(readString(conn.from)); // if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) {
auto outputPath = StorePath(readString(conn.from)); // throw Error("old-style build traces no longer supported");
store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); //}
} else {
auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn); auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
store->registerDrvOutput(realisation); store->registerDrvOutput(realisation);
}
logger->stopWork(); logger->stopWork();
break; break;
} }
case WorkerProto::Op::QueryRealisation: { case WorkerProto::Op::QueryRealisation: {
logger->startWork(); logger->startWork();
auto outputId = DrvOutput::parse(readString(conn.from)); auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
auto info = store->queryRealisation(outputId); std::optional<UnkeyedRealisation> info = *store->queryRealisation(outputId);
logger->stopWork(); logger->stopWork();
if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) {
std::set<StorePath> outPaths; std::set<StorePath> outPaths;
if (info) if (info)
outPaths.insert(info->outPath); outPaths.insert(info->outPath);
WorkerProto::write(*store, wconn, outPaths); WorkerProto::write(*store, wconn, outPaths);
} else if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) {
// No longer support this format
WorkerProto::write(*store, wconn, StringSet{});
} else { } else {
std::set<Realisation> realisations; WorkerProto::write(*store, wconn, info);
if (info)
realisations.insert({*info, outputId});
WorkerProto::write(*store, wconn, realisations);
} }
break; break;
} }

193
src/libstore/derivations.cc
View file

@ -866,13 +866,14 @@ DrvHashes drvHashes;
/* Look up the derivation by value and memoize the /* Look up the derivation by value and memoize the
`hashDerivationModulo` call. `hashDerivationModulo` call.
*/ */
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
{ {
std::optional<DrvHash> hash; std::optional<DrvHashModulo> hash;
if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) {
return *hash; return *hash;
} }
auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false);
// Cache it // Cache it
drvHashes.insert_or_assign(drvPath, h); drvHashes.insert_or_assign(drvPath, h);
return h; return h;
@ -895,12 +896,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa
don't leak the provenance of fixed outputs, reducing pointless cache don't leak the provenance of fixed outputs, reducing pointless cache
misses as the build itself won't know this. misses as the build itself won't know this.
*/ */
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{ {
auto type = drv.type();
/* Return a fixed hash for fixed-output derivations. */ /* Return a fixed hash for fixed-output derivations. */
if (type.isFixed()) { if (drv.type().isFixed()) {
std::map<std::string, Hash> outputHashes; std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) { for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw); auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
@ -910,54 +909,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
+ store.printStorePath(dof.path(store, drv.name, i.first))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
return DrvHash{ return outputHashes;
.hashes = outputHashes,
.kind = DrvHash::Kind::Regular,
};
} }
auto kind = std::visit( if (std::visit(
overloaded{ overloaded{
[](const DerivationType::InputAddressed & ia) { [](const DerivationType::InputAddressed & ia) {
/* This might be a "pesimistically" deferred output, so we don't /* This might be a "pesimistically" deferred output, so we don't
"taint" the kind yet. */ "taint" the kind yet. */
return DrvHash::Kind::Regular; return false;
}, },
[](const DerivationType::ContentAddressed & ca) { [](const DerivationType::ContentAddressed & ca) {
return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; // Already covered
assert(!ca.fixed);
return true;
}, },
[](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, [](const DerivationType::Impure &) { return true; }},
drv.type().raw); drv.type().raw)) {
return DrvHashModulo::DeferredDrv{};
}
/* For other derivations, replace the inputs paths with recursive
calls to this function. */
DerivedPathMap<StringSet>::ChildNode::Map inputs2; DerivedPathMap<StringSet>::ChildNode::Map inputs2;
for (auto & [drvPath, node] : drv.inputDrvs.map) { for (auto & [drvPath, node] : drv.inputDrvs.map) {
/* Need to build and resolve dynamic derivations first */
if (!node.childMap.empty()) {
return DrvHashModulo::DeferredDrv{};
}
const auto & res = pathDerivationModulo(store, drvPath); const auto & res = pathDerivationModulo(store, drvPath);
if (res.kind == DrvHash::Kind::Deferred) if (std::visit(
kind = DrvHash::Kind::Deferred; overloaded{
[&](const DrvHashModulo::DeferredDrv &) { return true; },
// Regular non-CA derivation, replace derivation
[&](const DrvHashModulo::DrvHash & drvHash) {
inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node);
return false;
},
// CA derivation's output hashes
[&](const DrvHashModulo::CaOutputHashes & outputHashes) {
for (auto & outputName : node.value) { for (auto & outputName : node.value) {
const auto h = get(res.hashes, outputName); /* Put each one in with a single "out" output.. */
const auto h = get(outputHashes, outputName);
if (!h) if (!h)
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); inputs2.insert_or_assign(
h->to_string(HashFormat::Base16, false),
DerivedPathMap<StringSet>::ChildNode{
.value = {"out"},
});
}
return false;
},
},
res.raw)) {
return DrvHashModulo::DeferredDrv{};
} }
} }
auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs) {
outputHashes.insert_or_assign(outputName, hash);
}
return DrvHash{
.hashes = outputHashes,
.kind = kind,
};
}
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
{
return hashDerivationModulo(store, drv, true).hashes;
} }
static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store)
@ -1107,22 +1118,39 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites)
} }
} }
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) void resolveInputAddressed(Store & store, Derivation & drv)
{ {
drv.applyRewrites(rewrites); std::optional<DrvHashModulo> hashModulo_;
auto hashModulo = [&]() -> const auto & {
if (!hashModulo_) {
// somewhat expensive so we do lazily
hashModulo_ = hashDerivationModulo(store, drv, true);
}
return *hashModulo_;
};
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) { for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) { if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) {
auto h = get(hashModulo.hashes, outputName); std::visit(
if (!h) overloaded{
throw Error( [&](const DrvHashModulo::DrvHash & drvHash) {
"derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); auto outPath = store.makeOutputPath(outputName, drvHash, drv.name);
auto outPath = store.makeOutputPath(outputName, *h, drv.name); drv.env.insert_or_assign(outputName, store.printStorePath(outPath));
drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput::InputAddressed{ output = DerivationOutput::InputAddressed{
.path = std::move(outPath), .path = std::move(outPath),
}; };
},
[&](const DrvHashModulo::CaOutputHashes &) {
/* Shouldn't happen as the original output is
deferred (waiting to be input-addressed). */
assert(false);
},
[&](const DrvHashModulo::DeferredDrv &) {
// Nothing to do, already deferred
},
},
hashModulo().raw);
} }
} }
} }
@ -1205,9 +1233,13 @@ std::optional<BasicDerivation> Derivation::tryResolve(
queryResolutionChain)) queryResolutionChain))
return std::nullopt; return std::nullopt;
rewriteDerivation(store, resolved, inputRewrites); resolved.applyRewrites(inputRewrites);
return resolved; Derivation resolved2{std::move(resolved)};
resolveInputAddressed(store, resolved2);
return resolved2;
} }
void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
@ -1235,46 +1267,79 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
// combinations that are currently prohibited. // combinations that are currently prohibited.
type(); type();
std::optional<DrvHash> hashesModulo; std::optional<DrvHashModulo> hashModulo_;
for (auto & i : outputs) {
auto hashModulo = [&]() -> const auto & {
if (!hashModulo_) {
// somewhat expensive so we do lazily
hashModulo_ = hashDerivationModulo(store, *this, true);
}
return *hashModulo_;
};
for (auto & [outputName, output] : outputs) {
std::visit( std::visit(
overloaded{ overloaded{
[&](const DerivationOutput::InputAddressed & doia) { [&](const DerivationOutput::InputAddressed & doia) {
if (!hashesModulo) { std::visit(
// somewhat expensive so we do lazily overloaded{
hashesModulo = hashDerivationModulo(store, *this, true); [&](const DrvHashModulo::DrvHash & drvHash) {
} StorePath recomputed = store.makeOutputPath(outputName, drvHash, drvName);
auto currentOutputHash = get(hashesModulo->hashes, i.first);
if (!currentOutputHash)
throw Error(
"derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
store.printStorePath(drvPath),
store.printStorePath(doia.path),
i.first);
StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName);
if (doia.path != recomputed) if (doia.path != recomputed)
throw Error( throw Error(
"derivation '%s' has incorrect output '%s', should be '%s'", "derivation '%s' has incorrect output '%s', should be '%s'",
store.printStorePath(drvPath), store.printStorePath(drvPath),
store.printStorePath(doia.path), store.printStorePath(doia.path),
store.printStorePath(recomputed)); store.printStorePath(recomputed));
envHasRightPath(doia.path, i.first); },
[&](const DrvHashModulo::CaOutputHashes &) {
/* Shouldn't happen as the original output is
input-addressed. */
assert(false);
},
[&](const DrvHashModulo::DeferredDrv &) {
throw Error(
"derivation '%s' has output '%s', but derivation is not yet ready to be input-addressed",
store.printStorePath(drvPath),
store.printStorePath(doia.path));
},
},
hashModulo().raw);
envHasRightPath(doia.path, outputName);
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
auto path = dof.path(store, drvName, i.first); auto path = dof.path(store, drvName, outputName);
envHasRightPath(path, i.first); envHasRightPath(path, outputName);
}, },
[&](const DerivationOutput::CAFloating &) { [&](const DerivationOutput::CAFloating &) {
/* Nothing to check */ /* Nothing to check */
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
/* Nothing to check */ /* Nothing to check */
std::visit(
overloaded{
[&](const DrvHashModulo::DrvHash & drvHash) {
throw Error(
"derivation '%s' has deferred output '%s', yet is ready to be input-addressed",
store.printStorePath(drvPath),
outputName);
},
[&](const DrvHashModulo::CaOutputHashes &) {
/* Shouldn't happen as the original output is
input-addressed. */
assert(false);
},
[&](const DrvHashModulo::DeferredDrv &) {
/* Nothing to check */
},
},
hashModulo().raw);
}, },
[&](const DerivationOutput::Impure &) { [&](const DerivationOutput::Impure &) {
/* Nothing to check */ /* Nothing to check */
}, },
}, },
i.second.raw); output.raw);
} }
} }

4
src/libstore/dummy-store.cc
View file

@ -336,7 +336,7 @@ struct DummyStoreImpl : DummyStore
void registerDrvOutput(const Realisation & output) override void registerDrvOutput(const Realisation & output) override
{ {
auto ref = make_ref<UnkeyedRealisation>(output); auto ref = make_ref<UnkeyedRealisation>(output);
buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, ref}}}, [&](auto & kv) { buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, ref}}}, [&](auto & kv) {
kv.second.insert_or_assign(output.id.outputName, make_ref<UnkeyedRealisation>(output)); kv.second.insert_or_assign(output.id.outputName, make_ref<UnkeyedRealisation>(output));
}); });
} }
@ -357,7 +357,7 @@ struct DummyStoreImpl : DummyStore
const DrvOutput & drvOutput, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override const DrvOutput & drvOutput, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override
{ {
bool visited = false; bool visited = false;
buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) {
if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) {
visited = true; visited = true;
callback(it->second.get_ptr()); callback(it->second.get_ptr());

9
src/libstore/include/nix/store/binary-cache-store.hh
View file

@ -82,8 +82,13 @@ protected:
/** /**
* The prefix under which realisation infos will be stored * The prefix under which realisation infos will be stored
*
* @note The previous (still experimental, though) hash-keyed
* realisations were under "realisations". "build trace" is a better
* name anyways (issue #11895), and this serves as some light
* versioning.
*/ */
constexpr const static std::string realisationsPrefix = "realisations"; constexpr const static std::string realisationsPrefix = "build-trace";
constexpr const static std::string cacheInfoFile = "nix-cache-info"; constexpr const static std::string cacheInfoFile = "nix-cache-info";
@ -92,7 +97,7 @@ protected:
/** /**
* Compute the path to the given realisation * Compute the path to the given realisation
* *
* It's `${realisationsPrefix}/${drvOutput}.doi`. * It's `${realisationsPrefix}/${drvPath}/${outputName}`.
*/ */
std::string makeRealisationPath(const DrvOutput & id); std::string makeRealisationPath(const DrvOutput & id);

1
src/libstore/include/nix/store/build/derivation-building-misc.hh
View file

@ -45,7 +45,6 @@ struct InitialOutputStatus
struct InitialOutput struct InitialOutput
{ {
Hash outputHash;
std::optional<InitialOutputStatus> known; std::optional<InitialOutputStatus> known;
}; };

2
src/libstore/include/nix/store/build/derivation-goal.hh
View file

@ -71,8 +71,6 @@ private:
*/ */
std::unique_ptr<Derivation> drv; std::unique_ptr<Derivation> drv;
const Hash outputHash;
const BuildMode buildMode; const BuildMode buildMode;
/** /**

5
src/libstore/include/nix/store/common-protocol-impl.hh
View file

@ -26,12 +26,13 @@ namespace nix {
LengthPrefixedProtoHelper<CommonProto, T>::write(store, conn, t); \ LengthPrefixedProtoHelper<CommonProto, T>::write(store, conn, t); \
} }
#define COMMA_ ,
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>) COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
#define COMMA_ ,
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T COMMA_ typename Compare>, std::set<T COMMA_ Compare>) COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T COMMA_ typename Compare>, std::set<T COMMA_ Compare>)
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>) COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename K COMMA_ typename V>, std::map<K COMMA_ V>) COMMON_USE_LENGTH_PREFIX_SERIALISER(
template<typename K COMMA_ typename V COMMA_ typename Compare>, std::map<K COMMA_ V COMMA_ Compare>)
#undef COMMA_ #undef COMMA_
/* protocol-specific templates */ /* protocol-specific templates */

7
src/libstore/include/nix/store/common-protocol.hh
View file

@ -12,7 +12,6 @@ struct Source;
class StorePath; class StorePath;
struct ContentAddress; struct ContentAddress;
struct DrvOutput; struct DrvOutput;
struct Realisation;
/** /**
* Shared serializers between the worker protocol, serve protocol, and a * Shared serializers between the worker protocol, serve protocol, and a
@ -70,8 +69,6 @@ template<>
DECLARE_COMMON_SERIALISER(ContentAddress); DECLARE_COMMON_SERIALISER(ContentAddress);
template<> template<>
DECLARE_COMMON_SERIALISER(DrvOutput); DECLARE_COMMON_SERIALISER(DrvOutput);
template<>
DECLARE_COMMON_SERIALISER(Realisation);
#define COMMA_ , #define COMMA_ ,
template<typename T> template<typename T>
@ -81,8 +78,8 @@ DECLARE_COMMON_SERIALISER(std::set<T COMMA_ Compare>);
template<typename... Ts> template<typename... Ts>
DECLARE_COMMON_SERIALISER(std::tuple<Ts...>); DECLARE_COMMON_SERIALISER(std::tuple<Ts...>);
template<typename K, typename V> template<typename K, typename V, typename Compare>
DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V>); DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
#undef COMMA_ #undef COMMA_
/** /**

55
src/libstore/include/nix/store/derivations.hh
View file

@ -427,35 +427,40 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName);
* derivations (fixed-output or not) will have a different hash for each * derivations (fixed-output or not) will have a different hash for each
* output. * output.
*/ */
struct DrvHash struct DrvHashModulo
{ {
/** /**
* Map from output names to hashes * Single hash for the derivation
*
* This is for an input-addressed derivation that doesn't
* transitively depend on any floating-CA derivations.
*/ */
std::map<std::string, Hash> hashes; using DrvHash = Hash;
enum struct Kind : bool {
/**
* Statically determined derivations.
* This hash will be directly used to compute the output paths
*/
Regular,
/** /**
* Floating-output derivations (and their reverse dependencies). * Known CA drv's output hashes, for fixed-output derivations whose
* output hashes are always known since they are fixed up-front.
*/ */
Deferred, using CaOutputHashes = std::map<std::string, Hash>;
};
/** /**
* The kind of derivation this is, simplified for just "derivation hash * This derivation doesn't yet have known output hashes.
* modulo" purposes. *
* Either because itself is floating CA, or it (transtively) depends
* on a floating CA derivation.
*/ */
Kind kind; using DeferredDrv = std::monostate;
using Raw = std::variant<DrvHash, CaOutputHashes, DeferredDrv>;
Raw raw;
bool operator==(const DrvHashModulo &) const = default;
// auto operator <=> (const DrvHashModulo &) const = default;
MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo);
}; };
void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
/** /**
* Returns hashes with the details of fixed-output subderivations * Returns hashes with the details of fixed-output subderivations
* expunged. * expunged.
@ -480,15 +485,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
* ATerm, after subderivations have been likewise expunged from that * ATerm, after subderivations have been likewise expunged from that
* derivation. * derivation.
*/ */
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/** /**
* Return a map associating each output to a hash that uniquely identifies its * If a derivation is input addressed and doesn't yet have its input
* derivation (modulo the self-references). * addressed (is deferred) try using `hashDerivationModulo`.
* *
* \todo What is the Hash in this map? * Does nothing if not deferred input-addressed, or
* `hashDerivationModulo` indicates it is missing inputs' output paths
* and is not yet ready (and must stay deferred).
*/ */
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv); void resolveInputAddressed(Store & store, Derivation & drv);
struct DrvHashFct struct DrvHashFct
{ {
@ -503,7 +510,7 @@ struct DrvHashFct
/** /**
* Memoisation of hashDerivationModulo(). * Memoisation of hashDerivationModulo().
*/ */
typedef boost::concurrent_flat_map<StorePath, DrvHash, DrvHashFct> DrvHashes; typedef boost::concurrent_flat_map<StorePath, DrvHashModulo, DrvHashFct> DrvHashes;
// FIXME: global, though at least thread-safe. // FIXME: global, though at least thread-safe.
extern DrvHashes drvHashes; extern DrvHashes drvHashes;

2
src/libstore/include/nix/store/dummy-store-impl.hh
View file

@ -49,7 +49,7 @@ struct DummyStore : virtual Store
* outer map for the derivation, and inner maps for the outputs of a * outer map for the derivation, and inner maps for the outputs of a
* given derivation. * given derivation.
*/ */
boost::concurrent_flat_map<Hash, std::map<std::string, ref<UnkeyedRealisation>>> buildTrace; boost::concurrent_flat_map<StorePath, std::map<std::string, ref<UnkeyedRealisation>>> buildTrace;
DummyStore(ref<const Config> config) DummyStore(ref<const Config> config)
: Store{*config} : Store{*config}

20
src/libstore/include/nix/store/length-prefixed-protocol-helper.hh
View file

@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector<T>);
#define COMMA_ , #define COMMA_ ,
template<class Inner, typename T, typename Compare> template<class Inner, typename T, typename Compare>
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T COMMA_ Compare>); LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T COMMA_ Compare>);
#undef COMMA_
template<class Inner, typename... Ts> template<class Inner, typename... Ts>
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>); LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>);
template<class Inner, typename K, typename V> template<class Inner, typename K, typename V, typename Compare>
#define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V> #define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V, Compare>
LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X);
#undef COMMA_
template<class Inner, typename T> template<class Inner, typename T>
std::vector<T> std::vector<T>
@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper<Inner, std::set<T, Compare>>::write(
} }
} }
template<class Inner, typename K, typename V> template<class Inner, typename K, typename V, typename Compare>
std::map<K, V> std::map<K, V, Compare> LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::read(
LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) const StoreDirConfig & store, typename Inner::ReadConn conn)
{ {
std::map<K, V> resMap; std::map<K, V, Compare> resMap;
auto size = readNum<size_t>(conn.from); auto size = readNum<size_t>(conn.from);
while (size--) { while (size--) {
auto k = S<K>::read(store, conn); auto k = S<K>::read(store, conn);
@ -123,9 +123,9 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(const StoreDirConfig & st
return resMap; return resMap;
} }
template<class Inner, typename K, typename V> template<class Inner, typename K, typename V, typename Compare>
void LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write( void LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::write(
const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap) const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V, Compare> & resMap)
{ {
conn.to << resMap.size(); conn.to << resMap.size();
for (auto & i : resMap) { for (auto & i : resMap) {

91
src/libstore/include/nix/store/realisation.hh
View file

@ -18,33 +18,40 @@ struct OutputsSpec;
/** /**
* A general `Realisation` key. * A general `Realisation` key.
* *
* This is similar to a `DerivedPath::Opaque`, but the derivation is * This is similar to a `DerivedPath::Built`, except it is only a single
* identified by its "hash modulo" instead of by its store path. * step: `drvPath` is a `StorePath` rather than a `DerivedPath`.
*/ */
struct DrvOutput struct DrvOutput
{ {
/** /**
* The hash modulo of the derivation. * The store path to the derivation
*
* Computed from the derivation itself for most types of
* derivations, but computed from the (fixed) content address of the
* output for fixed-output derivations.
*/ */
Hash drvHash; StorePath drvPath;
/** /**
* The name of the output. * The name of the output.
*/ */
OutputName outputName; OutputName outputName;
/**
* Skips the store dir on the `drvPath`
*/
std::string to_string() const; std::string to_string() const;
std::string strHash() const /**
{ * Skips the store dir on the `drvPath`
return drvHash.to_string(HashFormat::Base16, true); */
} static DrvOutput from_string(std::string_view);
static DrvOutput parse(const std::string &); /**
* Includes the store dir on `drvPath`
*/
std::string render(const StoreDirConfig & store) const;
/**
* Includes the store dir on `drvPath`
*/
static DrvOutput parse(const StoreDirConfig & store, std::string_view);
bool operator==(const DrvOutput &) const = default; bool operator==(const DrvOutput &) const = default;
auto operator<=>(const DrvOutput &) const = default; auto operator<=>(const DrvOutput &) const = default;
@ -56,14 +63,6 @@ struct UnkeyedRealisation
StringSet signatures; StringSet signatures;
/**
* The realisations that are required for the current one to be valid.
*
* When importing this realisation, the store will first check that all its
* dependencies exist, and map to the correct output path
*/
std::map<DrvOutput, StorePath> dependentRealisations;
std::string fingerprint(const DrvOutput & key) const; std::string fingerprint(const DrvOutput & key) const;
void sign(const DrvOutput & key, const Signer &); void sign(const DrvOutput & key, const Signer &);
@ -72,6 +71,16 @@ struct UnkeyedRealisation
size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const;
/**
* Just check the `outPath`. Signatures don't matter for this.
* Callers must ensure that the corresponding key is the same for
* most use-cases.
*/
bool isCompatibleWith(const UnkeyedRealisation & other) const
{
return outPath == other.outPath;
}
const StorePath & getPath() const const StorePath & getPath() const
{ {
return outPath; return outPath;
@ -85,12 +94,6 @@ struct Realisation : UnkeyedRealisation
{ {
DrvOutput id; DrvOutput id;
bool isCompatibleWith(const UnkeyedRealisation & other) const;
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
bool operator==(const Realisation &) const = default; bool operator==(const Realisation &) const = default;
auto operator<=>(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default;
}; };
@ -101,16 +104,7 @@ struct Realisation : UnkeyedRealisation
* Since these are the outputs of a single derivation, we know the * Since these are the outputs of a single derivation, we know the
* output names are unique so we can use them as the map key. * output names are unique so we can use them as the map key.
*/ */
typedef std::map<OutputName, Realisation> SingleDrvOutputs; typedef std::map<OutputName, UnkeyedRealisation> SingleDrvOutputs;
/**
* Collection type for multiple derivations' outputs' `Realisation`s.
*
* `DrvOutput` is used because in general the derivations are not all
* the same, so we need to identify firstly which derivation, and
* secondly which output of that derivation.
*/
typedef std::map<DrvOutput, Realisation> DrvOutputs;
struct OpaquePath struct OpaquePath
{ {
@ -154,10 +148,6 @@ struct RealisedPath
*/ */
const StorePath & path() const &; const StorePath & path() const &;
void closure(Store & store, Set & ret) const;
static void closure(Store & store, const Set & startPaths, Set & ret);
Set closure(Store & store) const;
bool operator==(const RealisedPath &) const = default; bool operator==(const RealisedPath &) const = default;
auto operator<=>(const RealisedPath &) const = default; auto operator<=>(const RealisedPath &) const = default;
}; };
@ -165,22 +155,21 @@ struct RealisedPath
class MissingRealisation : public Error class MissingRealisation : public Error
{ {
public: public:
MissingRealisation(DrvOutput & outputId) MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId)
: MissingRealisation(outputId.outputName, outputId.strHash()) : MissingRealisation(store, outputId.drvPath, outputId.outputName)
{ {
} }
MissingRealisation(std::string_view drv, OutputName outputName) MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName);
: Error( MissingRealisation(
"cannot operate on output '%s' of the " const StoreDirConfig & store,
"unbuilt derivation '%s'", const SingleDerivedPath & drvPath,
outputName, const StorePath & drvPathResolved,
drv) const OutputName & outputName);
{
}
}; };
} // namespace nix } // namespace nix
JSON_IMPL(nix::DrvOutput)
JSON_IMPL(nix::UnkeyedRealisation) JSON_IMPL(nix::UnkeyedRealisation)
JSON_IMPL(nix::Realisation) JSON_IMPL(nix::Realisation)

6
src/libstore/include/nix/store/serve-protocol-impl.hh
View file

@ -34,8 +34,10 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
#define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
SERVE_USE_LENGTH_PREFIX_SERIALISER( SERVE_USE_LENGTH_PREFIX_SERIALISER(
template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>, template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA
std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V>) typename Compare>
,
std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
/** /**
* Use `CommonProto` where possible. * Use `CommonProto` where possible.

15
src/libstore/include/nix/store/serve-protocol.hh
View file

@ -8,7 +8,7 @@ namespace nix {
#define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_1 0x390c9deb
#define SERVE_MAGIC_2 0x5452eecb #define SERVE_MAGIC_2 0x5452eecb
#define SERVE_PROTOCOL_VERSION (2 << 8 | 7) #define SERVE_PROTOCOL_VERSION (2 << 8 | 8)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -18,6 +18,9 @@ struct Source;
// items being serialised // items being serialised
struct BuildResult; struct BuildResult;
struct UnkeyedValidPathInfo; struct UnkeyedValidPathInfo;
struct DrvOutput;
struct UnkeyedRealisation;
struct Realisation;
/** /**
* The "serve protocol", used by ssh:// stores. * The "serve protocol", used by ssh:// stores.
@ -178,6 +181,12 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op)
template<> template<>
DECLARE_SERVE_SERIALISER(BuildResult); DECLARE_SERVE_SERIALISER(BuildResult);
template<> template<>
DECLARE_SERVE_SERIALISER(DrvOutput);
template<>
DECLARE_SERVE_SERIALISER(UnkeyedRealisation);
template<>
DECLARE_SERVE_SERIALISER(Realisation);
template<>
DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
template<> template<>
DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions);
@ -190,8 +199,8 @@ DECLARE_SERVE_SERIALISER(std::set<T COMMA_ Compare>);
template<typename... Ts> template<typename... Ts>
DECLARE_SERVE_SERIALISER(std::tuple<Ts...>); DECLARE_SERVE_SERIALISER(std::tuple<Ts...>);
template<typename K, typename V> template<typename K, typename V, typename Compare>
DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V>); DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
#undef COMMA_ #undef COMMA_
} // namespace nix } // namespace nix

3
src/libstore/include/nix/store/store-api.hh
View file

@ -1011,7 +1011,4 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashR
const ContentAddress * getDerivationCA(const BasicDerivation & drv); const ContentAddress * getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath>
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr);
} // namespace nix } // namespace nix

6
src/libstore/include/nix/store/worker-protocol-impl.hh
View file

@ -34,8 +34,10 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
#define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
WORKER_USE_LENGTH_PREFIX_SERIALISER( WORKER_USE_LENGTH_PREFIX_SERIALISER(
template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>, template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA
std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V>) typename Compare>
,
std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
/** /**
* Use `CommonProto` where possible. * Use `CommonProto` where possible.

17
src/libstore/include/nix/store/worker-protocol.hh
View file

@ -12,7 +12,7 @@ namespace nix {
/* Note: you generally shouldn't change the protocol version. Define a /* Note: you generally shouldn't change the protocol version. Define a
new `WorkerProto::Feature` instead. */ new `WorkerProto::Feature` instead. */
#define PROTOCOL_VERSION (1 << 8 | 38) #define PROTOCOL_VERSION (1 << 8 | 39)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -34,6 +34,9 @@ struct BuildResult;
struct KeyedBuildResult; struct KeyedBuildResult;
struct ValidPathInfo; struct ValidPathInfo;
struct UnkeyedValidPathInfo; struct UnkeyedValidPathInfo;
struct DrvOutput;
struct UnkeyedRealisation;
struct Realisation;
enum BuildMode : uint8_t; enum BuildMode : uint8_t;
enum TrustedFlag : bool; enum TrustedFlag : bool;
@ -258,6 +261,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo);
template<> template<>
DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo);
template<> template<>
DECLARE_WORKER_SERIALISER(DrvOutput);
template<>
DECLARE_WORKER_SERIALISER(UnkeyedRealisation);
template<>
DECLARE_WORKER_SERIALISER(Realisation);
template<>
DECLARE_WORKER_SERIALISER(std::optional<UnkeyedRealisation>);
template<>
DECLARE_WORKER_SERIALISER(BuildMode); DECLARE_WORKER_SERIALISER(BuildMode);
template<> template<>
DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>); DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>);
@ -274,8 +285,8 @@ DECLARE_WORKER_SERIALISER(std::set<T COMMA_ Compare>);
template<typename... Ts> template<typename... Ts>
DECLARE_WORKER_SERIALISER(std::tuple<Ts...>); DECLARE_WORKER_SERIALISER(std::tuple<Ts...>);
template<typename K, typename V> template<typename K, typename V, typename Compare>
DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V>); DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
#undef COMMA_ #undef COMMA_
} // namespace nix } // namespace nix

5
src/libstore/local-binary-cache-store.cc
View file

@ -58,9 +58,10 @@ protected:
std::shared_ptr<std::basic_iostream<char>> istream, std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override const std::string & mimeType) override
{ {
auto path2 = config->binaryCacheDir + "/" + path; auto path2 = std::filesystem::path{config->binaryCacheDir} / path;
static std::atomic<int> counter{0}; static std::atomic<int> counter{0};
Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); createDirs(path2.parent_path());
auto tmp = path2 + fmt(".tmp.%d.%d", getpid(), ++counter);
AutoDelete del(tmp, false); AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream); StreamToSourceAdapter source(istream);
writeFile(tmp, source); writeFile(tmp, source);

57
src/libstore/local-store.cc
View file

@ -110,8 +110,6 @@ struct LocalStore::State::Stmts
SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryAllRealisedOutputs;
SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryPathFromHashPart;
SQLiteStmt QueryValidPaths; SQLiteStmt QueryValidPaths;
SQLiteStmt QueryRealisationReferences;
SQLiteStmt AddRealisationReference;
}; };
LocalStore::LocalStore(ref<const Config> config) LocalStore::LocalStore(ref<const Config> config)
@ -390,21 +388,6 @@ LocalStore::LocalStore(ref<const Config> config)
where drvPath = ? where drvPath = ?
; ;
)"); )");
state->stmts->QueryRealisationReferences.create(
state->db,
R"(
select drvPath, outputName from Realisations
join RealisationsRefs on realisationReference = Realisations.id
where referrer = ?;
)");
state->stmts->AddRealisationReference.create(
state->db,
R"(
insert or replace into RealisationsRefs (referrer, realisationReference)
values (
(select id from Realisations where drvPath = ? and outputName = ?),
(select id from Realisations where drvPath = ? and outputName = ?));
)");
} }
} }
@ -636,7 +619,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
auto combinedSignatures = oldR->signatures; auto combinedSignatures = oldR->signatures;
combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); combinedSignatures.insert(info.signatures.begin(), info.signatures.end());
state->stmts->UpdateRealisedOutput state->stmts->UpdateRealisedOutput
.use()(concatStringsSep(" ", combinedSignatures))(info.id.strHash())(info.id.outputName) .use()(concatStringsSep(" ", combinedSignatures))(info.id.drvPath.to_string())(info.id.outputName)
.exec(); .exec();
} else { } else {
throw Error( throw Error(
@ -650,29 +633,10 @@ void LocalStore::registerDrvOutput(const Realisation & info)
} }
} else { } else {
state->stmts->RegisterRealisedOutput state->stmts->RegisterRealisedOutput
.use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( .use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))(
concatStringsSep(" ", info.signatures)) concatStringsSep(" ", info.signatures))
.exec(); .exec();
} }
for (auto & [outputId, depPath] : info.dependentRealisations) {
auto localRealisation = queryRealisationCore_(*state, outputId);
if (!localRealisation)
throw Error(
"unable to register the derivation '%s' as it "
"depends on the non existent '%s'",
info.id.to_string(),
outputId.to_string());
if (localRealisation->second.outPath != depPath)
throw Error(
"unable to register the derivation '%s' as it "
"depends on a realisation of '%s' that doesn’t"
"match what we have locally",
info.id.to_string(),
outputId.to_string());
state->stmts->AddRealisationReference
.use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName)
.exec();
}
}); });
} }
@ -1589,7 +1553,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
std::optional<std::pair<int64_t, UnkeyedRealisation>> std::optional<std::pair<int64_t, UnkeyedRealisation>>
LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id) LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id)
{ {
auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName)); auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName));
if (!useQueryRealisedOutput.next()) if (!useQueryRealisedOutput.next())
return std::nullopt; return std::nullopt;
auto realisationDbId = useQueryRealisedOutput.getInt(0); auto realisationDbId = useQueryRealisedOutput.getInt(0);
@ -1611,21 +1575,6 @@ std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(LocalStore
return std::nullopt; return std::nullopt;
auto [realisationDbId, res] = *maybeCore; auto [realisationDbId, res] = *maybeCore;
std::map<DrvOutput, StorePath> dependentRealisations;
auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId));
while (useRealisationRefs.next()) {
auto depId = DrvOutput{
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
useRealisationRefs.getStr(1),
};
auto dependentRealisation = queryRealisationCore_(state, depId);
assert(dependentRealisation); // Enforced by the db schema
auto outputPath = dependentRealisation->second.outPath;
dependentRealisations.insert({depId, outputPath});
}
res.dependentRealisations = dependentRealisations;
return {res}; return {res};
} }

68
src/libstore/misc.cc
View file

@ -239,16 +239,15 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
// If there are unknown output paths, attempt to find if the // If there are unknown output paths, attempt to find if the
// paths are known to substituters through a realisation. // paths are known to substituters through a realisation.
auto outputHashes = staticOutputHashes(*this, *drv);
knownOutputPaths = true; knownOutputPaths = true;
for (auto [outputName, hash] : outputHashes) { for (auto & [outputName, _] : drv->outputs) {
if (!bfd.outputs.contains(outputName)) if (!bfd.outputs.contains(outputName))
continue; continue;
bool found = false; bool found = false;
for (auto & sub : getDefaultSubstituters()) { for (auto & sub : getDefaultSubstituters()) {
auto realisation = sub->queryRealisation({hash, outputName}); auto realisation = sub->queryRealisation({drvPath, outputName});
if (!realisation) if (!realisation)
continue; continue;
found = true; found = true;
@ -329,65 +328,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
}}); }});
} }
std::map<DrvOutput, StorePath>
drvOutputReferences(const std::set<Realisation> & inputRealisations, const StorePathSet & pathReferences)
{
std::map<DrvOutput, StorePath> res;
for (const auto & input : inputRealisations) {
if (pathReferences.count(input.outPath)) {
res.insert({input.id, input.outPath});
}
}
return res;
}
std::map<DrvOutput, StorePath>
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : store;
std::set<Realisation> inputRealisations;
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty()) {
auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
for (const auto & outputName : inputNode.value) {
auto outputHash = get(outputHashes, outputName);
if (!outputHash)
throw Error(
"output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv));
DrvOutput key{*outputHash, outputName};
auto thisRealisation = store.queryRealisation(key);
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv));
inputRealisations.insert({*thisRealisation, std::move(key)});
}
}
if (!inputNode.value.empty()) {
auto d = makeConstantStorePathRef(inputDrv);
for (const auto & [outputName, childNode] : inputNode.childMap) {
SingleDerivedPath next = SingleDerivedPath::Built{d, outputName};
accumRealisations(
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
resolveDerivedPath(store, next, evalStore_),
childNode);
}
}
};
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
accumRealisations(inputDrv, inputNode);
auto info = store.queryPathInfo(outputPath);
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
{ {
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
@ -420,7 +360,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd,
OutputPathMap outputs; OutputPathMap outputs;
for (auto & [outputName, outputPathOpt] : outputsOpt) { for (auto & [outputName, outputPathOpt] : outputsOpt) {
if (!outputPathOpt) if (!outputPathOpt)
throw MissingRealisation(bfd.drvPath->to_string(store), outputName); throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName);
auto & outputPath = *outputPathOpt; auto & outputPath = *outputPathOpt;
outputs.insert_or_assign(outputName, outputPath); outputs.insert_or_assign(outputName, outputPath);
} }
@ -444,7 +384,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
bfd.output); bfd.output);
auto & optPath = outputPaths.at(bfd.output); auto & optPath = outputPaths.at(bfd.output);
if (!optPath) if (!optPath)
throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output);
return *optPath; return *optPath;
}, },
}, },

49
src/libstore/nar-info-disk-cache.cc
View file

@ -44,10 +44,16 @@ create table if not exists NARs (
create table if not exists Realisations ( create table if not exists Realisations (
cache integer not null, cache integer not null,
outputId text not null,
content blob, -- Json serialisation of the realisation, or null if the realisation is absent drvPath text not null,
outputName text not null,
-- The following are null if the realisation is absent
outputPath text,
sigs text,
timestamp integer not null, timestamp integer not null,
primary key (cache, outputId), primary key (cache, drvPath, outputName),
foreign key (cache) references BinaryCaches(id) on delete cascade foreign key (cache) references BinaryCaches(id) on delete cascade
); );
@ -121,24 +127,24 @@ public:
state->insertRealisation.create( state->insertRealisation.create(
state->db, state->db,
R"( R"(
insert or replace into Realisations(cache, outputId, content, timestamp) insert or replace into Realisations(cache, drvPath, outputName, outputPath, sigs, timestamp)
values (?, ?, ?, ?) values (?, ?, ?, ?, ?, ?)
)"); )");
state->insertMissingRealisation.create( state->insertMissingRealisation.create(
state->db, state->db,
R"( R"(
insert or replace into Realisations(cache, outputId, timestamp) insert or replace into Realisations(cache, drvPath, outputName, timestamp)
values (?, ?, ?) values (?, ?, ?, ?)
)"); )");
state->queryRealisation.create( state->queryRealisation.create(
state->db, state->db,
R"( R"(
select content from Realisations select outputPath, sigs from Realisations
where cache = ? and outputId = ? and where cache = ? and drvPath = ? and outputName = ? and
((content is null and timestamp > ?) or ((outputPath is null and timestamp > ?) or
(content is not null and timestamp > ?)) (outputPath is not null and timestamp > ?))
)"); )");
/* Periodically purge expired entries from the database. */ /* Periodically purge expired entries from the database. */
@ -295,22 +301,27 @@ public:
auto now = time(0); auto now = time(0);
auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(
now - settings.ttlNegativeNarInfoCache)(now - settings.ttlPositiveNarInfoCache)); now - settings.ttlNegativeNarInfoCache)(now - settings.ttlPositiveNarInfoCache));
if (!queryRealisation.next()) if (!queryRealisation.next())
return {oUnknown, 0}; return {oUnknown, nullptr};
if (queryRealisation.isNull(0)) if (queryRealisation.isNull(0))
return {oInvalid, 0}; return {oInvalid, nullptr};
try { try {
return { return {
oValid, oValid,
std::make_shared<Realisation>(nlohmann::json::parse(queryRealisation.getStr(0))), std::make_shared<Realisation>(
UnkeyedRealisation{
.outPath = StorePath{queryRealisation.getStr(0)},
.signatures = nlohmann::json::parse(queryRealisation.getStr(1)),
},
id),
}; };
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing the local disk cache"); e.addTrace({}, "reading build trace key-value from the local disk cache");
throw; throw;
} }
}); });
@ -355,7 +366,9 @@ public:
auto & cache(getCache(*state, uri)); auto & cache(getCache(*state, uri));
state->insertRealisation state->insertRealisation
.use()(cache.id)(realisation.id.to_string())(static_cast<nlohmann::json>(realisation).dump())(time(0)) .use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)(
realisation.outPath.to_string())(static_cast<nlohmann::json>(realisation.signatures).dump())(
time(0))
.exec(); .exec();
}); });
} }
@ -366,7 +379,7 @@ public:
auto state(_state.lock()); auto state(_state.lock());
auto & cache(getCache(*state, uri)); auto & cache(getCache(*state, uri));
state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec(); state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(0)).exec();
}); });
} }
}; };

165
src/libstore/realisation.cc
View file

@ -1,6 +1,5 @@
#include "nix/store/realisation.hh" #include "nix/store/realisation.hh"
#include "nix/store/store-api.hh" #include "nix/store/store-api.hh"
#include "nix/util/closure.hh"
#include "nix/util/signature/local-keys.hh" #include "nix/util/signature/local-keys.hh"
#include "nix/util/json-utils.hh" #include "nix/util/json-utils.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -9,63 +8,34 @@ namespace nix {
MakeError(InvalidDerivationOutputId, Error); MakeError(InvalidDerivationOutputId, Error);
DrvOutput DrvOutput::parse(const std::string & strRep) DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s)
{ {
size_t n = strRep.find("!"); size_t n = s.rfind('^');
if (n == strRep.npos) if (n == s.npos)
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s);
return DrvOutput{ return DrvOutput{
.drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), .drvPath = store.parseStorePath(s.substr(0, n)),
.outputName = strRep.substr(n + 1), .outputName = OutputName{s.substr(n + 1)},
}; };
} }
std::string DrvOutput::render(const StoreDirConfig & store) const
{
return std::string(store.printStorePath(drvPath)) + "^" + outputName;
}
std::string DrvOutput::to_string() const std::string DrvOutput::to_string() const
{ {
return strHash() + "!" + outputName; return std::string(drvPath.to_string()) + "^" + outputName;
}
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
{
std::set<Realisation> res;
Realisation::closure(store, startOutputs, res);
return res;
}
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
{
auto getDeps = [&](const Realisation & current) -> std::set<Realisation> {
std::set<Realisation> res;
for (auto & [currentDep, _] : current.dependentRealisations) {
if (auto currentRealisation = store.queryRealisation(currentDep))
res.insert({*currentRealisation, currentDep});
else
throw Error("Unrealised derivation '%s'", currentDep.to_string());
}
return res;
};
computeClosure<Realisation>(
startOutputs,
res,
[&](const Realisation & current, std::function<void(std::promise<std::set<Realisation>> &)> processEdges) {
std::promise<std::set<Realisation>> promise;
try {
auto res = getDeps(current);
promise.set_value(res);
} catch (...) {
promise.set_exception(std::current_exception());
}
return processEdges(promise);
});
} }
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
{ {
nlohmann::json serialized = Realisation{*this, key}; auto serialised = static_cast<nlohmann::json>(Realisation{*this, key});
serialized.erase("signatures"); auto value = serialised.find("value");
return serialized.dump(); assert(value != serialised.end());
value->erase("signatures");
return serialised.dump();
} }
void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer)
@ -97,45 +67,20 @@ const StorePath & RealisedPath::path() const &
return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw); return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw);
} }
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const MissingRealisation::MissingRealisation(
const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName)
: Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath))
{ {
if (outPath == other.outPath) {
if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
warn(
"Encountered a realisation for '%s' with an empty set of "
"dependencies. This is likely an artifact from an older Nix. "
"I’ll try to fix the realisation if I can",
id.to_string());
return true;
} else if (dependentRealisations == other.dependentRealisations) {
return true;
}
}
return false;
} }
void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret) MissingRealisation::MissingRealisation(
const StoreDirConfig & store,
const SingleDerivedPath & drvPath,
const StorePath & drvPathResolved,
const OutputName & outputName)
: MissingRealisation{store, drvPathResolved, outputName}
{ {
// FIXME: This only builds the store-path closure, not the real realisation addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store));
// closure
StorePathSet initialStorePaths, pathsClosure;
for (auto & path : startPaths)
initialStorePaths.insert(path.path());
store.computeFSClosure(initialStorePaths, pathsClosure);
ret.insert(startPaths.begin(), startPaths.end());
ret.insert(pathsClosure.begin(), pathsClosure.end());
}
void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const
{
RealisedPath::closure(store, {*this}, ret);
}
RealisedPath::Set RealisedPath::closure(Store & store) const
{
RealisedPath::Set ret;
closure(store, ret);
return ret;
} }
} // namespace nix } // namespace nix
@ -144,52 +89,62 @@ namespace nlohmann {
using namespace nix; using namespace nix;
UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & json0) DrvOutput adl_serializer<DrvOutput>::from_json(const json & json)
{ {
auto json = getObject(json0); auto obj = getObject(json);
return {
.drvPath = valueAt(obj, "drvPath"),
.outputName = getString(valueAt(obj, "outputName")),
};
}
void adl_serializer<DrvOutput>::to_json(json & json, const DrvOutput & drvOutput)
{
json = {
{"drvPath", drvOutput.drvPath},
{"outputName", drvOutput.outputName},
};
}
UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & json)
{
auto obj = getObject(json);
StringSet signatures; StringSet signatures;
if (auto signaturesOpt = optionalValueAt(json, "signatures")) if (auto * signaturesJson = get(obj, "signatures"))
signatures = *signaturesOpt; signatures = getStringSet(*signaturesJson);
std::map<DrvOutput, StorePath> dependentRealisations; return {
if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations")) .outPath = valueAt(obj, "outPath"),
for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies))
dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath});
return UnkeyedRealisation{
.outPath = valueAt(json, "outPath"),
.signatures = signatures, .signatures = signatures,
.dependentRealisations = dependentRealisations,
}; };
} }
void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r) void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
{ {
auto jsonDependentRealisations = nlohmann::json::object();
for (auto & [depId, depOutPath] : r.dependentRealisations)
jsonDependentRealisations.emplace(depId.to_string(), depOutPath);
json = { json = {
{"outPath", r.outPath}, {"outPath", r.outPath},
{"signatures", r.signatures}, {"signatures", r.signatures},
{"dependentRealisations", jsonDependentRealisations},
}; };
} }
Realisation adl_serializer<Realisation>::from_json(const json & json0) Realisation adl_serializer<Realisation>::from_json(const nlohmann::json & json)
{ {
auto json = getObject(json0); auto obj = getObject(json);
return Realisation{ return {
static_cast<UnkeyedRealisation>(json0), static_cast<UnkeyedRealisation>(valueAt(obj, "value")),
DrvOutput::parse(valueAt(json, "id")), static_cast<DrvOutput>(valueAt(obj, "key")),
}; };
} }
void adl_serializer<Realisation>::to_json(json & json, const Realisation & r) void adl_serializer<Realisation>::to_json(json & json, const Realisation & r)
{ {
json = static_cast<const UnkeyedRealisation &>(r); json = {
json["id"] = r.id.to_string(); {"key", r.id},
{"value", static_cast<const UnkeyedRealisation &>(r)},
};
} }
} // namespace nlohmann } // namespace nlohmann

38
src/libstore/remote-store.cc
View file

@ -507,7 +507,7 @@ void RemoteStore::queryRealisationUncached(
try { try {
auto conn(getConnection()); auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { if (GET_PROTOCOL_MINOR(conn->protoVersion) < 39) {
warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4");
return callback(nullptr); return callback(nullptr);
} }
@ -516,21 +516,12 @@ void RemoteStore::queryRealisationUncached(
conn->to << id.to_string(); conn->to << id.to_string();
conn.processStderr(); conn.processStderr();
auto real = [&]() -> std::shared_ptr<const UnkeyedRealisation> { callback([&]() -> std::shared_ptr<const UnkeyedRealisation> {
if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { auto realisation = WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(*this, *conn);
auto outPaths = WorkerProto::Serialise<std::set<StorePath>>::read(*this, *conn); if (!realisation)
if (outPaths.empty())
return nullptr; return nullptr;
return std::make_shared<const UnkeyedRealisation>(UnkeyedRealisation{.outPath = *outPaths.begin()}); return std::make_shared<const UnkeyedRealisation>(*realisation);
} else { }());
auto realisations = WorkerProto::Serialise<std::set<Realisation>>::read(*this, *conn);
if (realisations.empty())
return nullptr;
return std::make_shared<const UnkeyedRealisation>(*realisations.begin());
}
}();
callback(std::shared_ptr<const UnkeyedRealisation>(real));
} catch (...) { } catch (...) {
return callback.rethrow(); return callback.rethrow();
} }
@ -612,30 +603,19 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
OutputPathMap outputs; OutputPathMap outputs;
auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath);
auto drv = evalStore->readDerivation(drvPath);
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto built = resolveDerivedPath(*this, bfd, &*evalStore); auto built = resolveDerivedPath(*this, bfd, &*evalStore);
for (auto & [output, outputPath] : built) { for (auto & [output, outputPath] : built) {
auto outputHash = get(outputHashes, output); auto outputId = DrvOutput{drvPath, output};
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
printStorePath(drvPath),
output);
auto outputId = DrvOutput{*outputHash, output};
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
auto realisation = queryRealisation(outputId); auto realisation = queryRealisation(outputId);
if (!realisation) if (!realisation)
throw MissingRealisation(outputId); throw MissingRealisation(*this, outputId);
success.builtOutputs.emplace(output, Realisation{*realisation, outputId}); success.builtOutputs.emplace(output, *realisation);
} else { } else {
success.builtOutputs.emplace( success.builtOutputs.emplace(
output, output,
Realisation{
UnkeyedRealisation{ UnkeyedRealisation{
.outPath = outputPath, .outPath = outputPath,
},
outputId,
}); });
} }
} }

13
src/libstore/restricted-store.cc
View file

@ -281,9 +281,18 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
for (auto & result : results) { for (auto & result : results) {
if (auto * successP = result.tryGetSuccess()) { if (auto * successP = result.tryGetSuccess()) {
if (auto * pathBuilt = std::get_if<DerivedPathBuilt>(&result.path)) {
// TODO ugly extra IO
auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath);
for (auto & [outputName, output] : successP->builtOutputs) { for (auto & [outputName, output] : successP->builtOutputs) {
newPaths.insert(output.outPath); newPaths.insert(output.outPath);
newRealisations.insert(output); newRealisations.insert(
{output,
{
.drvPath = drvPath,
.outputName = outputName,
}});
}
} }
} }
} }
@ -292,7 +301,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
next->computeFSClosure(newPaths, closure); next->computeFSClosure(newPaths, closure);
for (auto & path : closure) for (auto & path : closure)
goal.addDependency(path); goal.addDependency(path);
for (auto & real : Realisation::closure(*next, newRealisations)) for (auto & real : newRealisations)
goal.addedDrvOutputs.insert(real.id); goal.addedDrvOutputs.insert(real.id);
return results; return results;

109
src/libstore/serve-protocol.cc
View file

@ -6,6 +6,7 @@
#include "nix/store/serve-protocol-impl.hh" #include "nix/store/serve-protocol-impl.hh"
#include "nix/util/archive.hh" #include "nix/util/archive.hh"
#include "nix/store/path-info.hh" #include "nix/store/path-info.hh"
#include "nix/util/json-utils.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -24,10 +25,19 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const StoreDirConfig & stor
if (GET_PROTOCOL_MINOR(conn.version) >= 3) if (GET_PROTOCOL_MINOR(conn.version) >= 3)
conn.from >> status.timesBuilt >> failure.isNonDeterministic >> status.startTime >> status.stopTime; conn.from >> status.timesBuilt >> failure.isNonDeterministic >> status.startTime >> status.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(store, conn); if (GET_PROTOCOL_MINOR(conn.version) >= 8) {
for (auto && [output, realisation] : builtOutputs) success.builtOutputs = ServeProto::Serialise<std::map<OutputName, UnkeyedRealisation>>::read(store, conn);
success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
for (auto & [output, realisation] : ServeProto::Serialise<StringMap>::read(store, conn)) {
size_t n = output.find("!");
if (n == output.npos)
throw Error("Invalid derivation output id %s", output);
success.builtOutputs.insert_or_assign(
output.substr(n + 1),
UnkeyedRealisation{
StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}});
}
} }
if (BuildResult::Success::statusIs(rawStatus)) { if (BuildResult::Success::statusIs(rawStatus)) {
@ -51,15 +61,18 @@ void ServeProto::Serialise<BuildResult>::write(
default value for the fields that don't exist in that case. */ default value for the fields that don't exist in that case. */
auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) {
conn.to << errorMsg; conn.to << errorMsg;
if (GET_PROTOCOL_MINOR(conn.version) >= 3) if (GET_PROTOCOL_MINOR(conn.version) >= 3)
conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
DrvOutputs builtOutputsFullKey; if (GET_PROTOCOL_MINOR(conn.version) >= 8) {
for (auto & [output, realisation] : builtOutputs) ServeProto::write(store, conn, builtOutputs);
builtOutputsFullKey.insert_or_assign(realisation.id, realisation); } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
ServeProto::write(store, conn, builtOutputsFullKey); // We no longer support these types of realisations
ServeProto::write(store, conn, StringMap{});
} }
}; };
std::visit( std::visit(
overloaded{ overloaded{
[&](const BuildResult::Failure & failure) { [&](const BuildResult::Failure & failure) {
@ -144,4 +157,82 @@ void ServeProto::Serialise<ServeProto::BuildOptions>::write(
} }
} }
UnkeyedRealisation ServeProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 8) {
throw Error(
"daemon protocol %d.%d is too old (< 2.8) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
auto outPath = ServeProto::Serialise<StorePath>::read(store, conn);
auto signatures = ServeProto::Serialise<StringSet>::read(store, conn);
return UnkeyedRealisation{
.outPath = std::move(outPath),
.signatures = std::move(signatures),
};
}
void ServeProto::Serialise<UnkeyedRealisation>::write(
const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 8) {
throw Error(
"daemon protocol %d.%d is too old (< 2.8) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
ServeProto::write(store, conn, info.outPath);
ServeProto::write(store, conn, info.signatures);
}
DrvOutput ServeProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 8) {
throw Error(
"daemon protocol %d.%d is too old (< 2.8) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
auto drvPath = ServeProto::Serialise<StorePath>::read(store, conn);
auto outputName = ServeProto::Serialise<std::string>::read(store, conn);
return DrvOutput{
.drvPath = std::move(drvPath),
.outputName = std::move(outputName),
};
}
void ServeProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 8) {
throw Error(
"daemon protocol %d.%d is too old (< 2.8) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
ServeProto::write(store, conn, info.drvPath);
ServeProto::write(store, conn, info.outputName);
}
Realisation ServeProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
{
auto id = ServeProto::Serialise<DrvOutput>::read(store, conn);
auto unkeyed = ServeProto::Serialise<UnkeyedRealisation>::read(store, conn);
return Realisation{
std::move(unkeyed),
std::move(id),
};
}
void ServeProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
{
ServeProto::write(store, conn, info.id);
ServeProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
}
} // namespace nix } // namespace nix

47
src/libstore/store-api.cc
View file

@ -360,9 +360,8 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore
return outputs; return outputs;
auto drv = evalStore.readInvalidDerivation(path); auto drv = evalStore.readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv); for (auto & [outputName, _] : drv.outputs) {
for (auto & [outputName, hash] : drvHashes) { auto realisation = queryRealisation(DrvOutput{path, outputName});
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation) { if (realisation) {
outputs.insert_or_assign(outputName, realisation->outPath); outputs.insert_or_assign(outputName, realisation->outPath);
} else { } else {
@ -382,7 +381,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev
OutputPathMap result; OutputPathMap result;
for (auto & [outName, optOutPath] : resp) { for (auto & [outName, optOutPath] : resp) {
if (!optOutPath) if (!optOutPath)
throw MissingRealisation(printStorePath(path), outName); throw MissingRealisation(*this, path, outName);
result.insert_or_assign(outName, *optOutPath); result.insert_or_assign(outName, *optOutPath);
} }
return result; return result;
@ -908,36 +907,21 @@ std::map<StorePath, StorePath> copyPaths(
SubstituteFlag substitute) SubstituteFlag substitute)
{ {
StorePathSet storePaths; StorePathSet storePaths;
std::set<Realisation> toplevelRealisations; std::vector<const Realisation *> realisations;
for (auto & path : paths) { for (auto & path : paths) {
storePaths.insert(path.path()); storePaths.insert(path.path());
if (auto * realisation = std::get_if<Realisation>(&path.raw)) { if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
experimentalFeatureSettings.require(Xp::CaDerivations); experimentalFeatureSettings.require(Xp::CaDerivations);
toplevelRealisations.insert(*realisation); realisations.push_back(realisation);
} }
} }
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
try { try {
// Copy the realisation closure // Copy the realisations. TODO batch this
processGraph<Realisation>( for (const auto * realisation : realisations)
Realisation::closure(srcStore, toplevelRealisations), dstStore.registerDrvOutput(*realisation, checkSigs);
[&](const Realisation & current) -> std::set<Realisation> {
std::set<Realisation> children;
for (const auto & [drvOutput, _] : current.dependentRealisations) {
auto currentChild = srcStore.queryRealisation(drvOutput);
if (!currentChild)
throw Error(
"incomplete realisation closure: '%s' is a "
"dependency of '%s' but isn't registered",
drvOutput.to_string(),
current.id.to_string());
children.insert({*currentChild, drvOutput});
}
return children;
},
[&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); });
} catch (MissingExperimentalFeature & e) { } catch (MissingExperimentalFeature & e) {
// Don't fail if the remote doesn't support CA derivations is it might // Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want // not be within our control to change that, and we might still want
@ -1048,8 +1032,19 @@ void copyClosure(
if (&srcStore == &dstStore) if (&srcStore == &dstStore)
return; return;
RealisedPath::Set closure; StorePathSet closure0;
RealisedPath::closure(srcStore, paths, closure); for (auto & path : paths) {
if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
closure0.insert(opaquePath->path);
}
}
StorePathSet closure1;
srcStore.computeFSClosure(closure0, closure1);
RealisedPath::Set closure = paths;
for (auto && path : closure1)
closure.insert({std::move(path)});
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
} }

5
src/libstore/unix/build/derivation-builder.cc
View file

@ -1870,7 +1870,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
{ {
.outPath = newInfo.path, .outPath = newInfo.path,
}, },
DrvOutput{oldinfo->outputHash, outputName}, DrvOutput{
.drvPath = drvPath,
.outputName = outputName,
},
}; };
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) {
store.signRealisation(thisRealisation); store.signRealisation(thisRealisation);

144
src/libstore/worker-protocol.cc
View file

@ -6,6 +6,7 @@
#include "nix/store/worker-protocol-impl.hh" #include "nix/store/worker-protocol-impl.hh"
#include "nix/util/archive.hh" #include "nix/util/archive.hh"
#include "nix/store/path-info.hh" #include "nix/store/path-info.hh"
#include "nix/util/json-utils.hh"
#include <chrono> #include <chrono>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -132,7 +133,7 @@ void WorkerProto::Serialise<DerivedPath>::write(
throw Error( throw Error(
"trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", "trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
store.printStorePath(drvPath), store.printStorePath(drvPath),
GET_PROTOCOL_MAJOR(conn.version), GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version)); GET_PROTOCOL_MINOR(conn.version));
}, },
[&](std::monostate) { [&](std::monostate) {
@ -174,14 +175,24 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & sto
if (GET_PROTOCOL_MINOR(conn.version) >= 29) { if (GET_PROTOCOL_MINOR(conn.version) >= 29) {
conn.from >> res.timesBuilt >> failure.isNonDeterministic >> res.startTime >> res.stopTime; conn.from >> res.timesBuilt >> failure.isNonDeterministic >> res.startTime >> res.stopTime;
} }
if (GET_PROTOCOL_MINOR(conn.version) >= 37) { if (GET_PROTOCOL_MINOR(conn.version) >= 37) {
res.cpuUser = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn); res.cpuUser = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
res.cpuSystem = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn); res.cpuSystem = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
} }
if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, conn); if (GET_PROTOCOL_MINOR(conn.version) >= 39) {
for (auto && [output, realisation] : builtOutputs) success.builtOutputs = WorkerProto::Serialise<std::map<OutputName, UnkeyedRealisation>>::read(store, conn);
success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
for (auto && [output, realisation] : WorkerProto::Serialise<StringMap>::read(store, conn)) {
size_t n = output.find("!");
if (n == output.npos)
throw Error("Invalid derivation output id %s", output);
success.builtOutputs.insert_or_assign(
output.substr(n + 1),
UnkeyedRealisation{
StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}});
}
} }
if (BuildResult::Success::statusIs(rawStatus)) { if (BuildResult::Success::statusIs(rawStatus)) {
@ -205,20 +216,24 @@ void WorkerProto::Serialise<BuildResult>::write(
default value for the fields that don't exist in that case. */ default value for the fields that don't exist in that case. */
auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) {
conn.to << errorMsg; conn.to << errorMsg;
if (GET_PROTOCOL_MINOR(conn.version) >= 29) { if (GET_PROTOCOL_MINOR(conn.version) >= 29) {
conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime;
} }
if (GET_PROTOCOL_MINOR(conn.version) >= 37) { if (GET_PROTOCOL_MINOR(conn.version) >= 37) {
WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuUser);
WorkerProto::write(store, conn, res.cpuSystem); WorkerProto::write(store, conn, res.cpuSystem);
} }
if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
DrvOutputs builtOutputsFullKey; if (GET_PROTOCOL_MINOR(conn.version) >= 39) {
for (auto & [output, realisation] : builtOutputs) WorkerProto::write(store, conn, builtOutputs);
builtOutputsFullKey.insert_or_assign(realisation.id, realisation); } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
WorkerProto::write(store, conn, builtOutputsFullKey); // Don't support those types of realisations anymore.
WorkerProto::write(store, conn, StringMap{});
} }
}; };
std::visit( std::visit(
overloaded{ overloaded{
[&](const BuildResult::Failure & failure) { [&](const BuildResult::Failure & failure) {
@ -309,4 +324,113 @@ void WorkerProto::Serialise<WorkerProto::ClientHandshakeInfo>::write(
} }
} }
UnkeyedRealisation WorkerProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error(
"daemon protocol %d.%d is too old (< 1.39) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
auto outPath = WorkerProto::Serialise<StorePath>::read(store, conn);
auto signatures = WorkerProto::Serialise<StringSet>::read(store, conn);
return UnkeyedRealisation{
.outPath = std::move(outPath),
.signatures = std::move(signatures),
};
}
void WorkerProto::Serialise<UnkeyedRealisation>::write(
const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error(
"daemon protocol %d.%d is too old (< 1.39) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
WorkerProto::write(store, conn, info.outPath);
WorkerProto::write(store, conn, info.signatures);
}
std::optional<UnkeyedRealisation>
WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
// Hack to improve compat
(void) WorkerProto::Serialise<std::string>::read(store, conn);
return std::nullopt;
} else {
auto temp = readNum<uint8_t>(conn.from);
switch (temp) {
case 0:
return std::nullopt;
case 1:
return WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
default:
throw Error("Invalid optional build trace from remote");
}
}
}
void WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::write(
const StoreDirConfig & store, WriteConn conn, const std::optional<UnkeyedRealisation> & info)
{
if (!info) {
conn.to << uint8_t{0};
} else {
conn.to << uint8_t{1};
WorkerProto::write(store, conn, *info);
}
}
DrvOutput WorkerProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error(
"daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
auto drvPath = WorkerProto::Serialise<StorePath>::read(store, conn);
auto outputName = WorkerProto::Serialise<std::string>::read(store, conn);
return DrvOutput{
.drvPath = std::move(drvPath),
.outputName = std::move(outputName),
};
}
void WorkerProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
{
if (GET_PROTOCOL_MINOR(conn.version) < 39) {
throw Error(
"daemon protocol %d.%d is too old (< 1.29) to understand build trace",
GET_PROTOCOL_MAJOR(conn.version) >> 8,
GET_PROTOCOL_MINOR(conn.version));
}
WorkerProto::write(store, conn, info.drvPath);
WorkerProto::write(store, conn, info.outputName);
}
Realisation WorkerProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
{
auto id = WorkerProto::Serialise<DrvOutput>::read(store, conn);
auto unkeyed = WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
return Realisation{
std::move(unkeyed),
std::move(id),
};
}
void WorkerProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
{
WorkerProto::write(store, conn, info.id);
WorkerProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
}
} // namespace nix } // namespace nix

6
src/nix/build-remote/build-remote.cc
View file

@ -346,13 +346,11 @@ static int main_build_remote(int argc, char ** argv)
optResult = std::move(res[0]); optResult = std::move(res[0]);
} }
auto outputHashes = staticOutputHashes(*store, drv);
std::set<Realisation> missingRealisations; std::set<Realisation> missingRealisations;
StorePathSet missingPaths; StorePathSet missingPaths;
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
for (auto & outputName : wantedOutputs) { for (auto & outputName : wantedOutputs) {
auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{*drvPath, outputName};
auto thisOutputId = DrvOutput{thisOutputHash, outputName};
if (!store->queryRealisation(thisOutputId)) { if (!store->queryRealisation(thisOutputId)) {
debug("missing output %s", outputName); debug("missing output %s", outputName);
assert(optResult); assert(optResult);
@ -362,7 +360,7 @@ static int main_build_remote(int argc, char ** argv)
auto i = success.builtOutputs.find(outputName); auto i = success.builtOutputs.find(outputName);
assert(i != success.builtOutputs.end()); assert(i != success.builtOutputs.end());
auto & newRealisation = i->second; auto & newRealisation = i->second;
missingRealisations.insert(newRealisation); missingRealisations.insert({newRealisation, thisOutputId});
missingPaths.insert(newRealisation.outPath); missingPaths.insert(newRealisation.outPath);
} }
} }

10
src/nix/develop.cc
View file

@ -279,16 +279,8 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
output.second = DerivationOutput::Deferred{}; output.second = DerivationOutput::Deferred{};
drv.env[output.first] = ""; drv.env[output.first] = "";
} }
auto hashesModulo = hashDerivationModulo(*evalStore, drv, true);
for (auto & output : drv.outputs) { resolveInputAddressed(*evalStore, drv);
Hash h = hashesModulo.hashes.at(output.first);
auto outPath = store->makeOutputPath(output.first, h, drv.name);
output.second = DerivationOutput::InputAddressed{
.path = outPath,
};
drv.env[output.first] = store->printStorePath(outPath);
}
} }
auto shellDrvPath = writeDerivation(*evalStore, drv); auto shellDrvPath = writeDerivation(*evalStore, drv);

7
src/perl/lib/Nix/Store.xs
View file

@ -163,10 +163,13 @@ StoreWrapper::queryPathInfo(char * path, int base32)
} }
SV * SV *
StoreWrapper::queryRawRealisation(char * outputId) StoreWrapper::queryRawRealisation(char * drvPath, char * outputName)
PPCODE: PPCODE:
try { try {
auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); auto realisation = THIS->store->queryRealisation(DrvOutput{
.drvPath = THIS->store->parseStorePath(drvPath),
.outputName = outputName,
});
if (realisation) if (realisation)
XPUSHs(sv_2mortal(newSVpv(static_cast<nlohmann::json>(*realisation).dump().c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(static_cast<nlohmann::json>(*realisation).dump().c_str(), 0)));
else else

5
tests/functional/ca/duplicate-realisation-in-closure.sh
View file

@ -25,4 +25,9 @@ nix build -f nondeterministic.nix dep2 --no-link
# If everything goes right, we should rebuild dep2 rather than fetch it from # If everything goes right, we should rebuild dep2 rather than fetch it from
# the cache (because that would mean duplicating `current-time` in the closure), # the cache (because that would mean duplicating `current-time` in the closure),
# and have `dep1 == dep2`. # and have `dep1 == dep2`.
# FIXME: Force the use of small-step resolutions only to fix this in a
# better way (#11896, #11928).
skipTest "temporarily broken because dependent realisations are removed"
nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link

11
tests/functional/ca/substitute.sh
View file

@ -22,7 +22,10 @@ nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix
# Restart the build on an empty store, ensuring that we don't build # Restart the build on an empty store, ensuring that we don't build
clearStore clearStore
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA # FIXME: `dependentCA` should not need to be explicitly mentioned in
# this. Force the use of small-step resolutions only to allow not
# mentioning it explicitly again. (#11896, #11928).
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA dependentCA
# Check that the thing we’ve just substituted has its realisation stored # Check that the thing we’ve just substituted has its realisation stored
nix realisation info --file ./content-addressed.nix transitivelyDependentCA nix realisation info --file ./content-addressed.nix transitivelyDependentCA
# Check that its dependencies have it too # Check that its dependencies have it too
@ -46,14 +49,14 @@ fi
clearStore clearStore
nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore"
clearStore clearStore
rm -r "$REMOTE_STORE_DIR/realisations" rm -r "$REMOTE_STORE_DIR/build-trace"
nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
# There's no easy way to check whether a realisation is present on the local # There's no easy way to check whether a realisation is present on the local
# store − short of manually querying the db, but the build environment doesn't # store − short of manually querying the db, but the build environment doesn't
# have the sqlite binary − so we instead push things again, and check that the # have the sqlite binary − so we instead push things again, and check that the
# realisations have correctly been pushed to the remote store # realisations have correctly been pushed to the remote store
nix copy --to "$REMOTE_STORE" --file ../simple.nix nix copy --to "$REMOTE_STORE" --file ../simple.nix
if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace")" ]]; then
echo "Realisations not rebuilt" echo "Realisations not rebuilt"
exit 1 exit 1
fi fi
@ -68,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
# Try rebuilding, but remove the realisations from the remote cache to force # Try rebuilding, but remove the realisations from the remote cache to force
# using the cachecache # using the cachecache
clearStore clearStore
rm "$REMOTE_STORE_DIR"/realisations/* rm -r "$REMOTE_STORE_DIR"/build-trace/*
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0