From 9abcc68ad1b1eabd03c56969d9df8b1330039817 Mon Sep 17 00:00:00 2001 From: Tristan Ross Date: Fri, 19 Sep 2025 09:48:08 -0700 Subject: [PATCH 001/201] libstore-c: add nix_store_get_fs_closure --- src/libstore-c/nix_api_store.cc | 30 ++++++++++++++++++++++++++++++ src/libstore-c/nix_api_store.h | 24 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index c4c17f127..6ee792fc3 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -126,6 +126,36 @@ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const c NIXC_CATCH_ERRS_NULL } +nix_err nix_store_get_fs_closure( + nix_c_context * context, + Store * store, + const StorePath * store_path, + bool flip_direction, + bool include_outputs, + bool include_derivers, + void * userdata, + void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path)) +{ + if (context) + context->last_err_code = NIX_OK; + try { + const auto nixStore = store->ptr; + + nix::StorePathSet set; + nixStore->computeFSClosure(store_path->path, set, flip_direction, include_outputs, include_derivers); + + if (callback) { + for (const auto & path : set) { + const StorePath tmp{path}; + callback(context, userdata, &tmp); + if (context && context->last_err_code != NIX_OK) + return context->last_err_code; + } + } + } + NIXC_CATCH_ERRS +} + nix_err nix_store_realise( nix_c_context * context, Store * store, diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index e76e376b4..fd7ce068a 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -245,6 +245,30 @@ void nix_derivation_free(nix_derivation * drv); */ nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path); +/** + * @brief Gets the closure of a specific store path + * + * @note The callback borrows each StorePath only for the duration of the call. + * + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] store_path The path to compute from + * @param[in] flip_direction + * @param[in] include_outputs + * @param[in] include_derivers + * @param[in] callback The function to call for every store path, in no particular order + * @param[in] userdata The userdata to pass to the callback + */ +nix_err nix_store_get_fs_closure( + nix_c_context * context, + Store * store, + const StorePath * store_path, + bool flip_direction, + bool include_outputs, + bool include_derivers, + void * userdata, + void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path)); + // cffi end #ifdef __cplusplus } From aace1fb5d698e763c7f4e3ebd04ea737631adc62 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Oct 2025 13:27:09 +0200 Subject: [PATCH 002/201] C API: test nix_store_get_fs_closure --- src/libstore-tests/nix_api_store.cc | 240 ++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index dfd554ec1..6d6017f1f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -218,6 +218,66 @@ struct LambdaAdapter } }; +class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base +{ +public: + StorePath * drvPath = nullptr; + nix_derivation * drv = nullptr; + Store * store = nullptr; + StorePath * outPath = nullptr; + + void SetUp() override + { + nix_api_store_test_base::SetUp(); + + nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::settings.substituters = {}; + + store = open_local_store(); + + std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; + std::stringstream buffer; + buffer << t.rdbuf(); + + drv = nix_derivation_from_json(ctx, store, buffer.str().c_str()); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + drvPath = nix_add_derivation(ctx, store, drv); + assert_ctx_ok(); + ASSERT_NE(drvPath, nullptr); + + auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath_) { + auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath_); + ASSERT_EQ(is_valid_path, true); + ASSERT_STREQ(outname, "out") << "Expected single 'out' output"; + ASSERT_EQ(outPath, nullptr) << "Output path callback should only be called once"; + outPath = nix_store_path_clone(outPath_); + }}; + + auto ret = nix_store_realise( + ctx, store, drvPath, static_cast(&cb), decltype(cb)::call_void); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + ASSERT_NE(outPath, nullptr) << "Derivation should have produced an output"; + } + + void TearDown() override + { + if (drvPath) + nix_store_path_free(drvPath); + if (outPath) + nix_store_path_free(outPath); + if (drv) + nix_derivation_free(drv); + if (store) + nix_store_free(store); + + nix_api_store_test_base::TearDown(); + } +}; + TEST_F(nix_api_store_test_base, build_from_json) { // FIXME get rid of these @@ -256,4 +316,184 @@ TEST_F(nix_api_store_test_base, build_from_json) nix_store_free(store); } +TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_with_outputs) +{ + // Test closure computation with include_outputs on a derivation path + struct CallbackData + { + std::set * paths; + }; + + std::set closure_paths; + CallbackData data{&closure_paths}; + + auto ret = nix_store_get_fs_closure( + ctx, + store, + drvPath, // Use derivation path + false, // flip_direction + true, // include_outputs - include the outputs in the closure + false, // include_derivers + &data, + [](nix_c_context * context, void * userdata, const StorePath * path) { + auto * data = static_cast(userdata); + std::string path_str; + nix_store_path_name(path, OBSERVE_STRING(path_str)); + auto [it, inserted] = data->paths->insert(path_str); + ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str; + }); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + + // The closure should contain the derivation and its outputs + ASSERT_GE(closure_paths.size(), 2); + + // Verify the output path is in the closure + std::string outPathName; + nix_store_path_name(outPath, OBSERVE_STRING(outPathName)); + ASSERT_EQ(closure_paths.count(outPathName), 1); +} + +TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_without_outputs) +{ + // Test closure computation WITHOUT include_outputs on a derivation path + struct CallbackData + { + std::set * paths; + }; + + std::set closure_paths; + CallbackData data{&closure_paths}; + + auto ret = nix_store_get_fs_closure( + ctx, + store, + drvPath, // Use derivation path + false, // flip_direction + false, // include_outputs - do NOT include the outputs + false, // include_derivers + &data, + [](nix_c_context * context, void * userdata, const StorePath * path) { + auto * data = static_cast(userdata); + std::string path_str; + nix_store_path_name(path, OBSERVE_STRING(path_str)); + auto [it, inserted] = data->paths->insert(path_str); + ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str; + }); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + + // Verify the output path is NOT in the closure + std::string outPathName; + nix_store_path_name(outPath, OBSERVE_STRING(outPathName)); + ASSERT_EQ(closure_paths.count(outPathName), 0) << "Output path should not be in closure when includeOutputs=false"; +} + +TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_flip_direction) +{ + // Test closure computation with flip_direction on a derivation path + // When flip_direction=true, we get the reverse dependencies (what depends on this path) + // For a derivation, this should NOT include outputs even with include_outputs=true + struct CallbackData + { + std::set * paths; + }; + + std::set closure_paths; + CallbackData data{&closure_paths}; + + auto ret = nix_store_get_fs_closure( + ctx, + store, + drvPath, // Use derivation path + true, // flip_direction - get reverse dependencies + true, // include_outputs + false, // include_derivers + &data, + [](nix_c_context * context, void * userdata, const StorePath * path) { + auto * data = static_cast(userdata); + std::string path_str; + nix_store_path_name(path, OBSERVE_STRING(path_str)); + auto [it, inserted] = data->paths->insert(path_str); + ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str; + }); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + + // Verify the output path is NOT in the closure when direction is flipped + std::string outPathName; + nix_store_path_name(outPath, OBSERVE_STRING(outPathName)); + ASSERT_EQ(closure_paths.count(outPathName), 0) << "Output path should not be in closure when flip_direction=true"; +} + +TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_include_derivers) +{ + // Test closure computation with include_derivers on an output path + // This should include the derivation that produced the output + struct CallbackData + { + std::set * paths; + }; + + std::set closure_paths; + CallbackData data{&closure_paths}; + + auto ret = nix_store_get_fs_closure( + ctx, + store, + outPath, // Use output path (not derivation) + false, // flip_direction + false, // include_outputs + true, // include_derivers - include the derivation + &data, + [](nix_c_context * context, void * userdata, const StorePath * path) { + auto * data = static_cast(userdata); + std::string path_str; + nix_store_path_name(path, OBSERVE_STRING(path_str)); + auto [it, inserted] = data->paths->insert(path_str); + ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str; + }); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + + // Verify the derivation path is in the closure + // Deriver is nasty stateful, and this assertion is only guaranteed because + // we're using an empty store as our starting point. Otherwise, if the + // output happens to exist, the deriver could be anything. + std::string drvPathName; + nix_store_path_name(drvPath, OBSERVE_STRING(drvPathName)); + ASSERT_EQ(closure_paths.count(drvPathName), 1) << "Derivation should be in closure when include_derivers=true"; +} + +TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagation) +{ + // Test that errors in the callback abort the closure computation + struct CallbackData + { + int * count; + }; + + int call_count = 0; + CallbackData data{&call_count}; + + auto ret = nix_store_get_fs_closure( + ctx, + store, + drvPath, // Use derivation path + false, // flip_direction + true, // include_outputs + false, // include_derivers + &data, + [](nix_c_context * context, void * userdata, const StorePath * path) { + auto * data = static_cast(userdata); + (*data->count)++; + // Set an error immediately + nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Test error"); + }); + + // Should have aborted with error + ASSERT_EQ(ret, NIX_ERR_UNKNOWN); + ASSERT_EQ(call_count, 1); // Should have been called exactly once, then aborted +} + } // namespace nixC From 3fb943d130868f2290d260bfd7a19cb633519ca9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Oct 2025 14:55:28 +0200 Subject: [PATCH 003/201] C API: Make store realise tests multi-platform ... and improve assertions. --- src/libstore-tests/nix_api_store.cc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 6d6017f1f..16d1ac0d8 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -240,7 +240,10 @@ public: std::stringstream buffer; buffer << t.rdbuf(); - drv = nix_derivation_from_json(ctx, store, buffer.str().c_str()); + // Replace the hardcoded system with the current system + std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get()); + + drv = nix_derivation_from_json(ctx, store, jsonStr.c_str()); assert_ctx_ok(); ASSERT_NE(drv, nullptr); @@ -249,6 +252,7 @@ public: ASSERT_NE(drvPath, nullptr); auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath_) { + ASSERT_NE(outname, nullptr) << "Output name should not be NULL"; auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath_); ASSERT_EQ(is_valid_path, true); ASSERT_STREQ(outname, "out") << "Expected single 'out' output"; @@ -292,7 +296,10 @@ TEST_F(nix_api_store_test_base, build_from_json) std::stringstream buffer; buffer << t.rdbuf(); - auto * drv = nix_derivation_from_json(ctx, store, buffer.str().c_str()); + // Replace the hardcoded system with the current system + std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get()); + + auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str()); assert_ctx_ok(); ASSERT_NE(drv, nullptr); @@ -300,15 +307,21 @@ TEST_F(nix_api_store_test_base, build_from_json) assert_ctx_ok(); ASSERT_NE(drv, nullptr); + int callbackCount = 0; auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { + ASSERT_NE(outname, nullptr); + ASSERT_STREQ(outname, "out"); + ASSERT_NE(outPath, nullptr); auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath); ASSERT_EQ(is_valid_path, true); + callbackCount++; }}; auto ret = nix_store_realise( ctx, store, drvPath, static_cast(&cb), decltype(cb)::call_void); assert_ctx_ok(); ASSERT_EQ(ret, NIX_OK); + ASSERT_EQ(callbackCount, 1) << "Callback should have been invoked exactly once"; // Clean up nix_store_path_free(drvPath); From 12293a8b1162bc273f991b098e05c93e5ff32c5f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Oct 2025 15:05:50 +0200 Subject: [PATCH 004/201] C API: Document nix_store_copy_closure flags --- src/libstore-c/nix_api_store.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index fd7ce068a..f477d084a 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -253,9 +253,14 @@ nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * @param[out] context Optional, stores error information * @param[in] store nix store reference * @param[in] store_path The path to compute from - * @param[in] flip_direction - * @param[in] include_outputs - * @param[in] include_derivers + * @param[in] flip_direction If false, compute the forward closure (paths referenced by any store path in the closure). + * If true, compute the backward closure (paths that reference any store path in the closure). + * @param[in] include_outputs If flip_direction is false: for any derivation in the closure, include its outputs. + * If flip_direction is true: for any output in the closure, include derivations that produce + * it. + * @param[in] include_derivers If flip_direction is false: for any output in the closure, include the derivation that + * produced it. + * If flip_direction is true: for any derivation in the closure, include its outputs. * @param[in] callback The function to call for every store path, in no particular order * @param[in] userdata The userdata to pass to the callback */ From 6fa03765edcce6e5403903cd68a2cc464e67e4d1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Oct 2025 15:19:40 +0200 Subject: [PATCH 005/201] C API: Propagate nix_store_realise build errors --- src/libstore-c/nix_api_store.cc | 8 ++ src/libstore-c/nix_api_store.h | 2 + src/libstore-tests/nix_api_store.cc | 135 ++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 6ee792fc3..e18463192 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -173,6 +173,14 @@ nix_err nix_store_realise( const auto nixStore = store->ptr; auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore); + assert(results.size() == 1); + + // Check if any builds failed + for (auto & result : results) { + if (!result.success()) + result.rethrow(); + } + if (callback) { for (const auto & result : results) { for (const auto & [outputName, realisation] : result.builtOutputs) { diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index f477d084a..964f6d6d5 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -186,6 +186,8 @@ nix_err nix_store_real_path( * @param[in] path Path to build * @param[in] userdata data to pass to every callback invocation * @param[in] callback called for every realised output + * @return NIX_OK if the build succeeded, or an error code if the build/scheduling/outputs/copying/etc failed. + * On error, the callback is never invoked and error information is stored in context. */ nix_err nix_store_realise( nix_c_context * context, diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 16d1ac0d8..045b4ad83 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -329,6 +329,141 @@ TEST_F(nix_api_store_test_base, build_from_json) nix_store_free(store); } +TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) +{ + // Test that nix_store_realise properly reports errors when the system is invalid + nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::settings.substituters = {}; + + auto * store = open_local_store(); + + std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; + std::stringstream buffer; + buffer << t.rdbuf(); + + // Use an invalid system that cannot be built + std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", "bogus65-bogusos"); + + auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str()); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + auto * drvPath = nix_add_derivation(ctx, store, drv); + assert_ctx_ok(); + ASSERT_NE(drvPath, nullptr); + + int callbackCount = 0; + auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }}; + + auto ret = nix_store_realise( + ctx, store, drvPath, static_cast(&cb), decltype(cb)::call_void); + + // Should fail with an error + ASSERT_NE(ret, NIX_OK); + ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build fails"; + + // Check that error message is set + std::string errMsg = nix_err_msg(nullptr, ctx, nullptr); + ASSERT_FALSE(errMsg.empty()) << "Error message should be set"; + ASSERT_NE(errMsg.find("system"), std::string::npos) << "Error should mention system"; + + // Clean up + nix_store_path_free(drvPath); + nix_derivation_free(drv); + nix_store_free(store); +} + +TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) +{ + // Test that nix_store_realise properly reports errors when the builder fails + nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::settings.substituters = {}; + + auto * store = open_local_store(); + + std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; + std::stringstream buffer; + buffer << t.rdbuf(); + + // Replace with current system and make builder command fail + std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get()); + jsonStr = nix::replaceStrings(jsonStr, "echo $name foo > $out", "exit 1"); + + auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str()); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + auto * drvPath = nix_add_derivation(ctx, store, drv); + assert_ctx_ok(); + ASSERT_NE(drvPath, nullptr); + + int callbackCount = 0; + auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }}; + + auto ret = nix_store_realise( + ctx, store, drvPath, static_cast(&cb), decltype(cb)::call_void); + + // Should fail with an error + ASSERT_NE(ret, NIX_OK); + ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build fails"; + + // Check that error message is set + std::string errMsg = nix_err_msg(nullptr, ctx, nullptr); + ASSERT_FALSE(errMsg.empty()) << "Error message should be set"; + + // Clean up + nix_store_path_free(drvPath); + nix_derivation_free(drv); + nix_store_free(store); +} + +TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output) +{ + // Test that nix_store_realise properly reports errors when builder succeeds but produces no output + nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::settings.substituters = {}; + + auto * store = open_local_store(); + + std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; + std::stringstream buffer; + buffer << t.rdbuf(); + + // Replace with current system and make builder succeed but not produce output + std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get()); + jsonStr = nix::replaceStrings(jsonStr, "echo $name foo > $out", "true"); + + auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str()); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + auto * drvPath = nix_add_derivation(ctx, store, drv); + assert_ctx_ok(); + ASSERT_NE(drvPath, nullptr); + + int callbackCount = 0; + auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }}; + + auto ret = nix_store_realise( + ctx, store, drvPath, static_cast(&cb), decltype(cb)::call_void); + + // Should fail with an error + ASSERT_NE(ret, NIX_OK); + ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build produces no output"; + + // Check that error message is set + std::string errMsg = nix_err_msg(nullptr, ctx, nullptr); + ASSERT_FALSE(errMsg.empty()) << "Error message should be set"; + + // Clean up + nix_store_path_free(drvPath); + nix_derivation_free(drv); + nix_store_free(store); +} + TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_with_outputs) { // Test closure computation with include_outputs on a derivation path From 6036aaf798f38a2c1a1d63a16f8566c98a60dbcf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Oct 2025 22:04:21 +0200 Subject: [PATCH 006/201] C API: Check output callback order --- src/libstore-tests/nix_api_store.cc | 149 ++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 045b4ad83..228b8069f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -613,6 +613,155 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_include_deriver ASSERT_EQ(closure_paths.count(drvPathName), 1) << "Derivation should be in closure when include_derivers=true"; } +TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) +{ + // Test that nix_store_realise returns outputs in alphabetical order by output name. + // This test uses a CA derivation with 10 outputs in randomized input order + // to verify that the callback order is deterministic and alphabetical. + nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::settings.substituters = {}; + + auto * store = open_local_store(); + + // Create a CA derivation with 10 outputs using proper placeholders + auto outa_ph = nix::hashPlaceholder("outa"); + auto outb_ph = nix::hashPlaceholder("outb"); + auto outc_ph = nix::hashPlaceholder("outc"); + auto outd_ph = nix::hashPlaceholder("outd"); + auto oute_ph = nix::hashPlaceholder("oute"); + auto outf_ph = nix::hashPlaceholder("outf"); + auto outg_ph = nix::hashPlaceholder("outg"); + auto outh_ph = nix::hashPlaceholder("outh"); + auto outi_ph = nix::hashPlaceholder("outi"); + auto outj_ph = nix::hashPlaceholder("outj"); + + std::string drvJson = R"({ + "version": 3, + "name": "multi-output-test", + "system": ")" + nix::settings.thisSystem.get() + + R"(", + "builder": "/bin/sh", + "args": ["-c", "echo a > $outa; echo b > $outb; echo c > $outc; echo d > $outd; echo e > $oute; echo f > $outf; echo g > $outg; echo h > $outh; echo i > $outi; echo j > $outj"], + "env": { + "builder": "/bin/sh", + "name": "multi-output-test", + "system": ")" + nix::settings.thisSystem.get() + + R"(", + "outf": ")" + outf_ph + + R"(", + "outd": ")" + outd_ph + + R"(", + "outi": ")" + outi_ph + + R"(", + "oute": ")" + oute_ph + + R"(", + "outh": ")" + outh_ph + + R"(", + "outc": ")" + outc_ph + + R"(", + "outb": ")" + outb_ph + + R"(", + "outg": ")" + outg_ph + + R"(", + "outj": ")" + outj_ph + + R"(", + "outa": ")" + outa_ph + + R"(" + }, + "inputDrvs": {}, + "inputSrcs": [], + "outputs": { + "outd": { "hashAlgo": "sha256", "method": "nar" }, + "outf": { "hashAlgo": "sha256", "method": "nar" }, + "outg": { "hashAlgo": "sha256", "method": "nar" }, + "outb": { "hashAlgo": "sha256", "method": "nar" }, + "outc": { "hashAlgo": "sha256", "method": "nar" }, + "outi": { "hashAlgo": "sha256", "method": "nar" }, + "outj": { "hashAlgo": "sha256", "method": "nar" }, + "outh": { "hashAlgo": "sha256", "method": "nar" }, + "outa": { "hashAlgo": "sha256", "method": "nar" }, + "oute": { "hashAlgo": "sha256", "method": "nar" } + } + })"; + + auto * drv = nix_derivation_from_json(ctx, store, drvJson.c_str()); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + auto * drvPath = nix_add_derivation(ctx, store, drv); + assert_ctx_ok(); + ASSERT_NE(drvPath, nullptr); + + // Realise the derivation - capture the order outputs are returned + std::map outputs; + std::vector output_order; + auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { + ASSERT_NE(outname, nullptr); + ASSERT_NE(outPath, nullptr); + output_order.push_back(outname); + outputs.emplace(outname, outPath->path); + }}; + + auto ret = nix_store_realise( + ctx, store, drvPath, static_cast(&cb), decltype(cb)::call_void); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + ASSERT_EQ(outputs.size(), 10); + + // Verify outputs are returned in alphabetical order by output name + std::vector expected_order = { + "outa", "outb", "outc", "outd", "oute", "outf", "outg", "outh", "outi", "outj"}; + ASSERT_EQ(output_order, expected_order) << "Outputs should be returned in alphabetical order by output name"; + + // Now compute closure with include_outputs and collect paths in order + struct CallbackData + { + std::vector * paths; + }; + + std::vector closure_paths; + CallbackData data{&closure_paths}; + + ret = nix_store_get_fs_closure( + ctx, + store, + drvPath, + false, // flip_direction + true, // include_outputs - include the outputs in the closure + false, // include_derivers + &data, + [](nix_c_context * context, void * userdata, const StorePath * path) { + auto * data = static_cast(userdata); + std::string path_str; + nix_store_path_name(path, OBSERVE_STRING(path_str)); + data->paths->push_back(path_str); + }); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + + // Should contain at least the derivation and 10 outputs + ASSERT_GE(closure_paths.size(), 11); + + // Verify all outputs are present in the closure + for (const auto & [outname, outPath] : outputs) { + std::string outPathName = store->ptr->printStorePath(outPath); + + bool found = false; + for (const auto & p : closure_paths) { + // nix_store_path_name returns just the name part, so match against full path name + if (outPathName.find(p) != std::string::npos) { + found = true; + break; + } + } + ASSERT_TRUE(found) << "Output " << outname << " (" << outPathName << ") not found in closure"; + } + + nix_store_path_free(drvPath); + nix_derivation_free(drv); + nix_store_free(store); +} + TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagation) { // Test that errors in the callback abort the closure computation From a48a737517af31683d331b0aadc990d15c214b34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Oct 2025 16:30:10 +0200 Subject: [PATCH 007/201] Use serializer for std::optional --- src/libstore/worker-protocol.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 4f7c28409..a17d2c028 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -251,11 +251,10 @@ void WorkerProto::Serialise::write( UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { - auto deriver = readString(conn.from); + auto deriver = WorkerProto::Serialise>::read(store, conn); auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); UnkeyedValidPathInfo info(narHash); - if (deriver != "") - info.deriver = store.parseStorePath(deriver); + info.deriver = std::move(deriver); info.references = WorkerProto::Serialise::read(store, conn); conn.from >> info.registrationTime >> info.narSize; if (GET_PROTOCOL_MINOR(conn.version) >= 16) { @@ -269,8 +268,8 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St void WorkerProto::Serialise::write( const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { - conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") - << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.deriver); + conn.to << pathInfo.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(store, conn, pathInfo.references); conn.to << pathInfo.registrationTime << pathInfo.narSize; if (GET_PROTOCOL_MINOR(conn.version) >= 16) { From d782c5e5863cdcfd3fc8013c84efa1053b3d2e80 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Oct 2025 16:57:43 +0200 Subject: [PATCH 008/201] Daemon protocol: Use the WorkerProto serializer for store paths --- src/libstore/daemon.cc | 56 +++++++++++++++++++----------------- src/libstore/remote-store.cc | 45 +++++++++++++++-------------- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 00c0a1fdd..d6d2a5781 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -312,7 +312,7 @@ static void performOp( switch (op) { case WorkerProto::Op::IsValidPath: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); @@ -339,7 +339,7 @@ static void performOp( } case WorkerProto::Op::HasSubstitutes: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet paths; // FIXME paths.insert(path); @@ -359,7 +359,7 @@ static void performOp( } case WorkerProto::Op::QueryPathHash: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); @@ -371,7 +371,7 @@ static void performOp( case WorkerProto::Op::QueryReferrers: case WorkerProto::Op::QueryValidDerivers: case WorkerProto::Op::QueryDerivationOutputs: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet paths; if (op == WorkerProto::Op::QueryReferences) @@ -389,7 +389,7 @@ static void performOp( } case WorkerProto::Op::QueryDerivationOutputNames: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto names = store->readDerivation(path).outputNames(); logger->stopWork(); @@ -398,7 +398,7 @@ static void performOp( } case WorkerProto::Op::QueryDerivationOutputMap: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); @@ -407,11 +407,11 @@ static void performOp( } case WorkerProto::Op::QueryDeriver: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto info = store->queryPathInfo(path); logger->stopWork(); - conn.to << (info->deriver ? store->printStorePath(*info->deriver) : ""); + WorkerProto::write(*store, conn, info->deriver); break; } @@ -420,7 +420,7 @@ static void performOp( logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - conn.to << (path ? store->printStorePath(*path) : ""); + WorkerProto::write(*store, conn, path); break; } @@ -505,7 +505,7 @@ static void performOp( store->addToStoreFromDump(*dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); - conn.to << store->printStorePath(path); + WorkerProto::write(*store, wconn, path); } break; } @@ -542,7 +542,7 @@ static void performOp( NoRepair); }); logger->stopWork(); - conn.to << store->printStorePath(path); + WorkerProto::write(*store, wconn, path); break; } @@ -591,7 +591,7 @@ static void performOp( } case WorkerProto::Op::BuildDerivation: { - auto drvPath = store->parseStorePath(readString(conn.from)); + auto drvPath = WorkerProto::Serialise::read(*store, rconn); BasicDerivation drv; /* * Note: unlike wopEnsurePath, this operation reads a @@ -668,7 +668,7 @@ static void performOp( } case WorkerProto::Op::EnsurePath: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); store->ensurePath(path); logger->stopWork(); @@ -677,7 +677,7 @@ static void performOp( } case WorkerProto::Op::AddTempRoot: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); store->addTempRoot(path); logger->stopWork(); @@ -733,8 +733,10 @@ static void performOp( conn.to << size; for (auto & [target, links] : roots) - for (auto & link : links) - conn.to << link << store->printStorePath(target); + for (auto & link : links) { + conn.to << link; + WorkerProto::write(*store, wconn, target); + } break; } @@ -799,7 +801,7 @@ static void performOp( } case WorkerProto::Op::QuerySubstitutablePathInfo: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); @@ -808,7 +810,8 @@ static void performOp( if (i == infos.end()) conn.to << 0; else { - conn.to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); + conn.to << 1; + WorkerProto::write(*store, wconn, i->second.deriver); WorkerProto::write(*store, wconn, i->second.references); conn.to << i->second.downloadSize << i->second.narSize; } @@ -829,8 +832,8 @@ static void performOp( logger->stopWork(); conn.to << infos.size(); for (auto & i : infos) { - conn.to << store->printStorePath(i.first) - << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + WorkerProto::write(*store, wconn, i.first); + WorkerProto::write(*store, wconn, i.second.deriver); WorkerProto::write(*store, wconn, i.second.references); conn.to << i.second.downloadSize << i.second.narSize; } @@ -846,7 +849,7 @@ static void performOp( } case WorkerProto::Op::QueryPathInfo: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); std::shared_ptr info; logger->startWork(); info = store->queryPathInfo(path); @@ -880,7 +883,7 @@ static void performOp( } case WorkerProto::Op::AddSignatures: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); StringSet sigs = readStrings(conn.from); logger->startWork(); store->addSignatures(path, sigs); @@ -890,7 +893,7 @@ static void performOp( } case WorkerProto::Op::NarFromPath: { - auto path = store->parseStorePath(readString(conn.from)); + auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); logger->stopWork(); dumpPath(store->toRealPath(path), conn.to); @@ -899,12 +902,11 @@ static void performOp( case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; - auto path = store->parseStorePath(readString(conn.from)); - auto deriver = readString(conn.from); + auto path = WorkerProto::Serialise::read(*store, rconn); + auto deriver = WorkerProto::Serialise>::read(*store, rconn); auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); ValidPathInfo info{path, narHash}; - if (deriver != "") - info.deriver = store->parseStorePath(deriver); + info.deriver = std::move(deriver); info.references = WorkerProto::Serialise::read(*store, rconn); conn.from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(conn.from); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8dd5bc064..0d83aed4c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -159,7 +159,8 @@ void RemoteStore::setOptions() bool RemoteStore::isValidPathUncached(const StorePath & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::IsValidPath << printStorePath(path); + conn->to << WorkerProto::Op::IsValidPath; + WorkerProto::write(*this, *conn, path); conn.processStderr(); return readInt(conn->from); } @@ -205,10 +206,8 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); - auto deriver = readString(conn->from); - if (deriver != "") - info.deriver = parseStorePath(deriver); + SubstitutablePathInfo & info(infos[WorkerProto::Serialise::read(*this, *conn)]); + info.deriver = WorkerProto::Serialise>::read(*this, *conn); info.references = WorkerProto::Serialise::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); @@ -235,7 +234,8 @@ void RemoteStore::queryPathInfoUncached( void RemoteStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryReferrers << printStorePath(path); + conn->to << WorkerProto::Op::QueryReferrers; + WorkerProto::write(*this, *conn, path); conn.processStderr(); for (auto & i : WorkerProto::Serialise::read(*this, *conn)) referrers.insert(i); @@ -244,7 +244,8 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet & referrer StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryValidDerivers << printStorePath(path); + conn->to << WorkerProto::Op::QueryValidDerivers; + WorkerProto::write(*this, *conn, path); conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); } @@ -255,7 +256,8 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) return Store::queryDerivationOutputs(path); } auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path); + conn->to << WorkerProto::Op::QueryDerivationOutputs; + WorkerProto::write(*this, *conn, path); conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); } @@ -266,7 +268,8 @@ RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * eva if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { if (!evalStore_) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); + conn->to << WorkerProto::Op::QueryDerivationOutputMap; + WorkerProto::write(*this, *conn, path); conn.processStderr(); return WorkerProto::Serialise>>::read(*this, *conn); } else { @@ -299,10 +302,7 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & auto conn(getConnection()); conn->to << WorkerProto::Op::QueryPathFromHashPart << hashPart; conn.processStderr(); - Path path = readString(conn->from); - if (path.empty()) - return {}; - return parseStorePath(path); + return WorkerProto::Serialise>::read(*this, *conn); } ref RemoteStore::addCAToStore( @@ -384,7 +384,7 @@ ref RemoteStore::addCAToStore( break; } } - auto path = parseStorePath(readString(conn->from)); + auto path = WorkerProto::Serialise::read(*this, *conn); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); return queryPathInfo(path); @@ -426,9 +426,10 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(HashFormat::Base16, false); + conn->to << WorkerProto::Op::AddToStoreNar; + WorkerProto::write(*this, *conn, info.path); + WorkerProto::write(*this, *conn, info.deriver); + conn->to << info.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -663,7 +664,8 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD void RemoteStore::ensurePath(const StorePath & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::EnsurePath << printStorePath(path); + conn->to << WorkerProto::Op::EnsurePath; + WorkerProto::write(*this, *conn, path); conn.processStderr(); readInt(conn->from); } @@ -683,8 +685,7 @@ Roots RemoteStore::findRoots(bool censor) Roots result; while (count--) { Path link = readString(conn->from); - auto target = parseStorePath(readString(conn->from)); - result[std::move(target)].emplace(link); + result[WorkerProto::Serialise::read(*this, *conn)].emplace(link); } return result; } @@ -728,7 +729,9 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddSignatures << printStorePath(storePath) << sigs; + conn->to << WorkerProto::Op::AddSignatures; + WorkerProto::write(*this, *conn, storePath); + conn->to << sigs; conn.processStderr(); readInt(conn->from); } From b1d067c9bb33bbe35507872636d5e1c499b4ea7c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Oct 2025 20:34:15 +0300 Subject: [PATCH 009/201] tests/nixos: Rename back S3 store nixos test --- ci/gha/tests/default.nix | 2 +- tests/nixos/default.nix | 2 +- ...curl-s3-binary-cache-store.nix => s3-binary-cache-store.nix} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/nixos/{curl-s3-binary-cache-store.nix => s3-binary-cache-store.nix} (100%) diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index fac4f9002..2bfdae17b 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -222,7 +222,7 @@ rec { }; vmTests = { - inherit (nixosTests) curl-s3-binary-cache-store; + inherit (nixosTests) s3-binary-cache-store; } // lib.optionalAttrs (!withSanitizers && !withCoverage) { # evalNixpkgs uses non-instrumented components from hydraJobs, so only run it diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 0112d2e2f..edfa4124f 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -199,7 +199,7 @@ in user-sandboxing = runNixOSTest ./user-sandboxing; - curl-s3-binary-cache-store = runNixOSTest ./curl-s3-binary-cache-store.nix; + s3-binary-cache-store = runNixOSTest ./s3-binary-cache-store.nix; fsync = runNixOSTest ./fsync.nix; diff --git a/tests/nixos/curl-s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix similarity index 100% rename from tests/nixos/curl-s3-binary-cache-store.nix rename to tests/nixos/s3-binary-cache-store.nix From dc03c6a8121a42f597268dcfbee2087a5a80018d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Oct 2025 21:13:04 +0300 Subject: [PATCH 010/201] libstore: Put all the AWS credentials logic behind interface class AwsCredentialProvider This makes it so we don't need to rely on global variables and hacky destructors to clean up another global variable. Just putting it in the correct order in the class is more than enough. --- src/libstore/aws-creds.cc | 88 +++++++------------ src/libstore/filetransfer.cc | 22 ++--- src/libstore/include/nix/store/aws-creds.hh | 46 ++++++---- src/libstore/unix/build/derivation-builder.cc | 2 +- 4 files changed, 69 insertions(+), 89 deletions(-) diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index 93fc3da33..4ba5b7dee 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -24,50 +24,6 @@ namespace nix { namespace { -// Global credential provider cache using boost's concurrent map -// Key: profile name (empty string for default profile) -using CredentialProviderCache = - boost::concurrent_flat_map>; - -static CredentialProviderCache credentialProviderCache; - -/** - * Clear all cached credential providers. - * Called automatically by CrtWrapper destructor during static destruction. - */ -static void clearAwsCredentialsCache() -{ - credentialProviderCache.clear(); -} - -static void initAwsCrt() -{ - struct CrtWrapper - { - Aws::Crt::ApiHandle apiHandle; - - CrtWrapper() - { - apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast(nullptr)); - } - - ~CrtWrapper() - { - try { - // CRITICAL: Clear credential provider cache BEFORE AWS CRT shuts down - // This ensures all providers (which hold references to ClientBootstrap) - // are destroyed while AWS CRT is still valid - clearAwsCredentialsCache(); - // Now it's safe for ApiHandle destructor to run - } catch (...) { - ignoreExceptionInDestructor(); - } - } - }; - - static CrtWrapper crt; -} - static AwsCredentials getCredentialsFromProvider(std::shared_ptr provider) { if (!provider || !provider->IsValid()) { @@ -113,7 +69,35 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr(nullptr)); + } + + AwsCredentials getCredentialsRaw(const std::string & profile); + + AwsCredentials getCredentials(const ParsedS3URL & url) override + { + auto profile = url.profile.value_or(""); + try { + return getCredentialsRaw(profile); + } catch (AwsAuthError & e) { + warn("AWS authentication failed for S3 request %s: %s", url.toHttpsUrl(), e.what()); + credentialProviderCache.erase(profile); + throw; + } + } + +private: + Aws::Crt::ApiHandle apiHandle; + boost::concurrent_flat_map> + credentialProviderCache; +}; + +AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile) { // Get or create credential provider with caching std::shared_ptr provider; @@ -132,8 +116,6 @@ AwsCredentials getAwsCredentials(const std::string & profile) profile.empty() ? "(default)" : profile.c_str()); try { - initAwsCrt(); - if (profile.empty()) { Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config; config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); @@ -173,17 +155,15 @@ AwsCredentials getAwsCredentials(const std::string & profile) return getCredentialsFromProvider(provider); } -void invalidateAwsCredentials(const std::string & profile) +ref makeAwsCredentialsProvider() { - credentialProviderCache.erase(profile); + return make_ref(); } -AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url) +ref getAwsCredentialsProvider() { - std::string profile = s3Url.profile.value_or(""); - - // Get credentials (automatically cached) - return getAwsCredentials(profile); + static auto instance = makeAwsCredentialsProvider(); + return instance; } } // namespace nix diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 981d49d77..201f2984e 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -883,22 +883,12 @@ void FileTransferRequest::setupForS3() if (usernameAuth) { debug("Using pre-resolved AWS credentials from parent process"); sessionToken = preResolvedAwsSessionToken; - } else { - std::string profile = parsedS3.profile.value_or(""); - try { - auto creds = getAwsCredentials(profile); - usernameAuth = UsernameAuth{ - .username = creds.accessKeyId, - .password = creds.secretAccessKey, - }; - sessionToken = creds.sessionToken; - } catch (const AwsAuthError & e) { - warn("AWS authentication failed for S3 request %s: %s", uri, e.what()); - // Invalidate the cached credentials so next request will retry - invalidateAwsCredentials(profile); - // Continue without authentication - might be a public bucket - return; - } + } else if (auto creds = getAwsCredentialsProvider()->maybeGetCredentials(parsedS3)) { + usernameAuth = UsernameAuth{ + .username = creds->accessKeyId, + .password = creds->secretAccessKey, + }; + sessionToken = creds->sessionToken; } if (sessionToken) headers.emplace_back("x-amz-security-token", *sessionToken); diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh index 6e653936c..d72290ced 100644 --- a/src/libstore/include/nix/store/aws-creds.hh +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -5,6 +5,7 @@ #if NIX_WITH_AWS_AUTH # include "nix/store/s3-url.hh" +# include "nix/util/ref.hh" # include "nix/util/error.hh" # include @@ -38,30 +39,39 @@ struct AwsCredentials */ MakeError(AwsAuthError, Error); -/** - * Get AWS credentials for the given profile. - * This function automatically caches credential providers to avoid - * creating multiple providers for the same profile. - * - * @param profile The AWS profile name (empty string for default profile) - * @return AWS credentials - * @throws AwsAuthError if credentials cannot be resolved - */ -AwsCredentials getAwsCredentials(const std::string & profile = ""); +class AwsCredentialProvider +{ +public: + /** + * Get AWS credentials for the given URL. + * + * @param url The S3 url to get the credentials for + * @return AWS credentials + * @throws AwsAuthError if credentials cannot be resolved + */ + virtual AwsCredentials getCredentials(const ParsedS3URL & url) = 0; + + std::optional maybeGetCredentials(const ParsedS3URL & url) + { + try { + return getCredentials(url); + } catch (AwsAuthError & e) { + return std::nullopt; + } + } + + virtual ~AwsCredentialProvider() {} +}; /** - * Invalidate cached credentials for a profile (e.g., on authentication failure). - * The next request for this profile will create a new provider. - * - * @param profile The AWS profile name to invalidate + * Create a new instancee of AwsCredentialProvider. */ -void invalidateAwsCredentials(const std::string & profile); +ref makeAwsCredentialsProvider(); /** - * Pre-resolve AWS credentials for S3 URLs. - * Used to cache credentials in parent process before forking. + * Get a reference to the global AwsCredentialProvider. */ -AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url); +ref getAwsCredentialsProvider(); } // namespace nix #endif diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 8a0fa5ef7..1246fbf26 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -958,7 +958,7 @@ std::optional DerivationBuilderImpl::preResolveAwsCredentials() auto s3Url = ParsedS3URL::parse(parsedUrl); // Use the preResolveAwsCredentials from aws-creds - auto credentials = nix::preResolveAwsCredentials(s3Url); + auto credentials = getAwsCredentialsProvider()->getCredentials(s3Url); debug("Successfully pre-resolved AWS credentials in parent process"); return credentials; } From 33e94fe19fdedca5dd89fdc0b292938ac58dc81a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Oct 2025 21:48:13 +0300 Subject: [PATCH 011/201] libstore: Make AwsAuthError more legible Instead of the cryptic: > error: Failed to resolve AWS credentials: error code 6153` We now get more legible: > error: AWS authentication error: 'Valid credentials could not be sourced by the IMDS provider' (6153) --- src/libstore/aws-creds.cc | 9 +++++++-- src/libstore/include/nix/store/aws-creds.hh | 17 +++++++++++++---- src/libstore/meson.build | 2 ++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index 4ba5b7dee..6c9bc99b2 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -22,6 +22,12 @@ namespace nix { +AwsAuthError::AwsAuthError(int errorCode) + : Error("AWS authentication error: '%s' (%d)", aws_error_str(errorCode), errorCode) + , errorCode(errorCode) +{ +} + namespace { static AwsCredentials getCredentialsFromProvider(std::shared_ptr provider) @@ -35,8 +41,7 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptrGetCredentials([prom](std::shared_ptr credentials, int errorCode) { if (errorCode != 0 || !credentials) { - prom->set_exception( - std::make_exception_ptr(AwsAuthError("Failed to resolve AWS credentials: error code %d", errorCode))); + prom->set_exception(std::make_exception_ptr(AwsAuthError(errorCode))); } else { auto accessKeyId = Aws::Crt::ByteCursorToStringView(credentials->GetAccessKeyId()); auto secretAccessKey = Aws::Crt::ByteCursorToStringView(credentials->GetSecretAccessKey()); diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh index d72290ced..30f6592a0 100644 --- a/src/libstore/include/nix/store/aws-creds.hh +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -34,10 +34,19 @@ struct AwsCredentials } }; -/** - * Exception thrown when AWS authentication fails - */ -MakeError(AwsAuthError, Error); +class AwsAuthError : public Error +{ + std::optional errorCode; + +public: + using Error::Error; + AwsAuthError(int errorCode); + + std::optional getErrorCode() const + { + return errorCode; + } +}; class AwsCredentialProvider { diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 78a3dd9b3..40da06e6b 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -158,6 +158,8 @@ curl_s3_store_opt = get_option('curl-s3-store').require( if curl_s3_store_opt.enabled() deps_other += aws_crt_cpp + aws_c_common = cxx.find_library('aws-c-common', required : true) + deps_other += aws_c_common endif configdata_pub.set('NIX_WITH_AWS_AUTH', curl_s3_store_opt.enabled().to_int()) From e7047fde2549aa207ebd28cfb67a7eb21471c708 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Oct 2025 21:33:42 +0300 Subject: [PATCH 012/201] libstore: Remove the unnecessary 'error: ' prefix in warning message --- src/libstore/aws-creds.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index 6c9bc99b2..d58293560 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -90,7 +90,7 @@ public: try { return getCredentialsRaw(profile); } catch (AwsAuthError & e) { - warn("AWS authentication failed for S3 request %s: %s", url.toHttpsUrl(), e.what()); + warn("AWS authentication failed for S3 request %s: %s", url.toHttpsUrl(), e.message()); credentialProviderCache.erase(profile); throw; } From 1c02dd5b9c2b65115c49d2dbed43c01d467f77c2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Oct 2025 15:49:47 -0400 Subject: [PATCH 013/201] Allow for standard nlohmann JSON serializers to take separate XP features I realized that we can actually do this thing, even though it is not what nlohmann expects at all, because the extra parameter has a default argument so nlohmann doesn't need to care. Sneaky! --- src/libstore-tests/derivation.cc | 36 +++++++------------ src/libstore/derivations.cc | 9 ++--- src/libstore/include/nix/store/derivations.hh | 4 +-- .../nix/util/tests/json-characterization.hh | 4 +-- src/libutil/include/nix/util/json-impls.hh | 14 ++++++++ 5 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/libstore-tests/derivation.cc b/src/libstore-tests/derivation.cc index 65a5d011d..75bf75753 100644 --- a/src/libstore-tests/derivation.cc +++ b/src/libstore-tests/derivation.cc @@ -66,23 +66,17 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) FormatError); } -#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \ - TEST_P(FIXTURE, from_json) \ - { \ - const auto & [name, expected] = GetParam(); \ - /* Don't use readJsonTest because we want to check experimental \ - features. */ \ - readTest(Path{"output-"} + name + ".json", [&](const auto & encoded_) { \ - json j = json::parse(encoded_); \ - DerivationOutput got = DerivationOutput::fromJSON(j, mockXpSettings); \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_P(FIXTURE, to_json) \ - { \ - const auto & [name, value] = GetParam(); \ - writeJsonTest("output-" + name, value); \ +#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \ + TEST_P(FIXTURE, from_json) \ + { \ + const auto & [name, expected] = GetParam(); \ + readJsonTest(Path{"output-"} + name, expected, mockXpSettings); \ + } \ + \ + TEST_P(FIXTURE, to_json) \ + { \ + const auto & [name, value] = GetParam(); \ + writeJsonTest("output-" + name, value); \ } struct DerivationOutputJsonTest : DerivationTest, @@ -193,13 +187,7 @@ INSTANTIATE_TEST_SUITE_P( TEST_P(FIXTURE, from_json) \ { \ const auto & drv = GetParam(); \ - /* Don't use readJsonTest because we want to check experimental \ - features. */ \ - readTest(drv.name + ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - Derivation got = Derivation::fromJSON(encoded, mockXpSettings); \ - ASSERT_EQ(got, drv); \ - }); \ + readJsonTest(drv.name, drv, mockXpSettings); \ } \ \ TEST_P(FIXTURE, to_json) \ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 24dd61807..d39080e08 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1496,9 +1496,10 @@ namespace nlohmann { using namespace nix; -DerivationOutput adl_serializer::from_json(const json & json) +DerivationOutput +adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) { - return DerivationOutput::fromJSON(json); + return DerivationOutput::fromJSON(json, xpSettings); } void adl_serializer::to_json(json & json, const DerivationOutput & c) @@ -1506,9 +1507,9 @@ void adl_serializer::to_json(json & json, const DerivationOutp json = c.toJSON(); } -Derivation adl_serializer::from_json(const json & json) +Derivation adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) { - return Derivation::fromJSON(json); + return Derivation::fromJSON(json, xpSettings); } void adl_serializer::to_json(json & json, const Derivation & c) diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 0dfb80347..45188d6b3 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -537,5 +537,5 @@ std::string hashPlaceholder(const OutputNameView outputName); } // namespace nix -JSON_IMPL(nix::DerivationOutput) -JSON_IMPL(nix::Derivation) +JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput) +JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index 5a38b8e2c..d713c615b 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -24,12 +24,12 @@ struct JsonCharacterizationTest : virtual CharacterizationTest * @param test hook that takes the contents of the file and does the * actual work */ - void readJsonTest(PathView testStem, const T & expected) + void readJsonTest(PathView testStem, const T & expected, auto... args) { using namespace nlohmann; readTest(Path{testStem} + ".json", [&](const auto & encodedRaw) { auto encoded = json::parse(encodedRaw); - T decoded = adl_serializer::from_json(encoded); + T decoded = adl_serializer::from_json(encoded, args...); ASSERT_EQ(decoded, expected); }); } diff --git a/src/libutil/include/nix/util/json-impls.hh b/src/libutil/include/nix/util/json-impls.hh index 751fc410f..802c212e1 100644 --- a/src/libutil/include/nix/util/json-impls.hh +++ b/src/libutil/include/nix/util/json-impls.hh @@ -3,6 +3,8 @@ #include +#include "nix/util/experimental-features.hh" + // Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types #define JSON_IMPL(TYPE) \ namespace nlohmann { \ @@ -14,3 +16,15 @@ static void to_json(json & json, const TYPE & t); \ }; \ } + +#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \ + namespace nlohmann { \ + using namespace nix; \ + template<> \ + struct adl_serializer \ + { \ + static TYPE \ + from_json(const json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); \ + static void to_json(json & json, const TYPE & t); \ + }; \ + } From a2c6f38e1fd1869be638756549de949c315a845b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Oct 2025 16:08:59 -0400 Subject: [PATCH 014/201] Remove now-redundant methods for JSON on `Derivation` --- .../derivation-advanced-attrs.cc | 78 +++++++++---------- src/libstore-tests/derivation.cc | 3 +- src/libstore/derivations.cc | 72 ++++++----------- src/libstore/include/nix/store/derivations.hh | 11 --- src/nix/derivation-add.cc | 2 +- src/nix/derivation-show.cc | 2 +- 6 files changed, 67 insertions(+), 101 deletions(-) diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 9c13bf048..02bc8fa24 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -14,7 +14,7 @@ namespace nix { -using nlohmann::json; +using namespace nlohmann; class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest { @@ -51,44 +51,44 @@ using BothFixtures = ::testing::TypesreadTest(NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ - Derivation got = Derivation::fromJSON(encoded, this->mockXpSettings); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ - { \ - this->writeTest( \ - NAME ".json", \ - [&]() -> json { \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings).toJSON(); \ - }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ - { \ - this->readTest(NAME ".drv", [&](auto encoded) { \ - /* Use JSON file instead of C++ literal as source of truth. */ \ - auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ - auto expected = Derivation::fromJSON(json, this->mockXpSettings); \ - auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ - EXPECT_EQ(got.toJSON(), expected.toJSON()); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ +#define TEST_ATERM_JSON(STEM, NAME) \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ + { \ + this->readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ + Derivation got = adl_serializer::from_json(encoded, this->mockXpSettings); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ + { \ + this->writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ + { \ + this->readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto j = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ + auto expected = adl_serializer::from_json(j, this->mockXpSettings); \ + auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ + EXPECT_EQ(static_cast(got), static_cast(expected)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ /* No corresponding write test, because we need to read the drv to write the json file */ TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); diff --git a/src/libstore-tests/derivation.cc b/src/libstore-tests/derivation.cc index 75bf75753..6b33e5442 100644 --- a/src/libstore-tests/derivation.cc +++ b/src/libstore-tests/derivation.cc @@ -201,7 +201,8 @@ INSTANTIATE_TEST_SUITE_P( const auto & drv = GetParam(); \ readTest(drv.name + ".drv", [&](auto encoded) { \ auto got = parseDerivation(*store, std::move(encoded), drv.name, mockXpSettings); \ - ASSERT_EQ(got.toJSON(), drv.toJSON()); \ + using nlohmann::json; \ + ASSERT_EQ(static_cast(got), static_cast(drv)); \ ASSERT_EQ(got, drv); \ }); \ } \ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d39080e08..f44bf3e70 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1261,9 +1261,15 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); -nlohmann::json DerivationOutput::toJSON() const +} // namespace nix + +namespace nlohmann { + +using namespace nix; + +void adl_serializer::to_json(json & res, const DerivationOutput & o) { - nlohmann::json res = nlohmann::json::object(); + res = nlohmann::json::object(); std::visit( overloaded{ [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, @@ -1289,12 +1295,11 @@ nlohmann::json DerivationOutput::toJSON() const res["impure"] = true; }, }, - raw); - return res; + o.raw); } DerivationOutput -DerivationOutput::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) +adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) { std::set keys; auto & json = getObject(_json); @@ -1362,18 +1367,18 @@ DerivationOutput::fromJSON(const nlohmann::json & _json, const ExperimentalFeatu } } -nlohmann::json Derivation::toJSON() const +void adl_serializer::to_json(json & res, const Derivation & d) { - nlohmann::json res = nlohmann::json::object(); + res = nlohmann::json::object(); - res["name"] = name; + res["name"] = d.name; res["version"] = 3; { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); - for (auto & [outputName, output] : outputs) { + for (auto & [outputName, output] : d.outputs) { outputsObj[outputName] = output; } } @@ -1381,7 +1386,7 @@ nlohmann::json Derivation::toJSON() const { auto & inputsList = res["inputSrcs"]; inputsList = nlohmann::json ::array(); - for (auto & input : inputSrcs) + for (auto & input : d.inputSrcs) inputsList.emplace_back(input); } @@ -1401,24 +1406,22 @@ nlohmann::json Derivation::toJSON() const { auto & inputDrvsObj = res["inputDrvs"]; inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : inputDrvs.map) { + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); } } } - res["system"] = platform; - res["builder"] = builder; - res["args"] = args; - res["env"] = env; + res["system"] = d.platform; + res["builder"] = d.builder; + res["args"] = d.args; + res["env"] = d.env; - if (structuredAttrs) - res["structuredAttrs"] = structuredAttrs->structuredAttrs; - - return res; + if (d.structuredAttrs) + res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -Derivation Derivation::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) +Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) { using nlohmann::detail::value_t; @@ -1434,7 +1437,7 @@ Derivation Derivation::fromJSON(const nlohmann::json & _json, const Experimental try { auto outputs = getObject(valueAt(json, "outputs")); for (auto & [outputName, output] : outputs) { - res.outputs.insert_or_assign(outputName, DerivationOutput::fromJSON(output, xpSettings)); + res.outputs.insert_or_assign(outputName, adl_serializer::from_json(output, xpSettings)); } } catch (Error & e) { e.addTrace({}, "while reading key 'outputs'"); @@ -1490,31 +1493,4 @@ Derivation Derivation::fromJSON(const nlohmann::json & _json, const Experimental return res; } -} // namespace nix - -namespace nlohmann { - -using namespace nix; - -DerivationOutput -adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) -{ - return DerivationOutput::fromJSON(json, xpSettings); -} - -void adl_serializer::to_json(json & json, const DerivationOutput & c) -{ - json = c.toJSON(); -} - -Derivation adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) -{ - return Derivation::fromJSON(json, xpSettings); -} - -void adl_serializer::to_json(json & json, const Derivation & c) -{ - json = c.toJSON(); -} - } // namespace nlohmann diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 45188d6b3..4615d8acd 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -134,13 +134,6 @@ struct DerivationOutput */ std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - - nlohmann::json toJSON() const; - /** - * @param xpSettings Stop-gap to avoid globals during unit tests. - */ - static DerivationOutput - fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; typedef std::map DerivationOutputs; @@ -390,10 +383,6 @@ struct Derivation : BasicDerivation { } - nlohmann::json toJSON() const; - static Derivation - fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - bool operator==(const Derivation &) const = default; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. // auto operator <=> (const Derivation &) const = default; diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 2d13aba52..48e935092 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -33,7 +33,7 @@ struct CmdAddDerivation : MixDryRun, StoreCommand { auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); - auto drv = Derivation::fromJSON(json); + auto drv = static_cast(json); auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 20e54bba7..1528f5b51 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -58,7 +58,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON if (!drvPath.isDerivation()) continue; - jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath).toJSON(); + jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath); } printJSON(jsonRoot); } From 1177d65094acd4a7b9d57f2672c08fd72d007d99 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Oct 2025 16:44:27 -0400 Subject: [PATCH 015/201] Properly check xp features when deserializing deriving paths --- src/libstore-tests/derived-path.cc | 74 +++++++++++++------ src/libstore/derived-path.cc | 23 ++++-- .../include/nix/store/derived-path.hh | 8 +- 3 files changed, 69 insertions(+), 36 deletions(-) diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index 6e7648f25..70e789c0c 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -3,13 +3,13 @@ #include #include -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/json-characterization.hh" #include "nix/store/tests/derived-path.hh" #include "nix/store/tests/libstore.hh" namespace nix { -class DerivedPathTest : public CharacterizationTest, public LibStoreTest +class DerivedPathTest : public virtual CharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "derived-path"; @@ -123,25 +123,51 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o)) using nlohmann::json; -#define TEST_JSON(TYPE, NAME, VAL) \ - static const TYPE NAME = VAL; \ - \ - TEST_F(DerivedPathTest, NAME##_from_json) \ - { \ - readTest(#NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - TYPE got = static_cast(encoded); \ - ASSERT_EQ(got, NAME); \ - }); \ - } \ - \ - TEST_F(DerivedPathTest, NAME##_to_json) \ - { \ - writeTest( \ - #NAME ".json", \ - [&]() -> json { return static_cast(NAME); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ +struct SingleDerivedPathJsonTest : DerivedPathTest, + JsonCharacterizationTest, + ::testing::WithParamInterface +{}; + +struct DerivedPathJsonTest : DerivedPathTest, + JsonCharacterizationTest, + ::testing::WithParamInterface +{}; + +#define TEST_JSON(TYPE, NAME, VAL) \ + static const TYPE NAME = VAL; \ + \ + TEST_F(TYPE##JsonTest, NAME##_from_json) \ + { \ + readJsonTest(#NAME, NAME); \ + } \ + \ + TEST_F(TYPE##JsonTest, NAME##_to_json) \ + { \ + writeJsonTest(#NAME, NAME); \ + } + +#define TEST_JSON_XP_DYN(TYPE, NAME, VAL) \ + static const TYPE NAME = VAL; \ + \ + TEST_F(TYPE##JsonTest, NAME##_from_json_throws_without_xp) \ + { \ + std::optional ret; \ + readTest(#NAME ".json", [&](const auto & encoded_) { ret = json::parse(encoded_); }); \ + if (ret) { \ + EXPECT_THROW(nlohmann::adl_serializer::from_json(*ret), MissingExperimentalFeature); \ + } \ + } \ + \ + TEST_F(TYPE##JsonTest, NAME##_from_json) \ + { \ + ExperimentalFeatureSettings xpSettings; \ + xpSettings.set("experimental-features", "dynamic-derivations"); \ + readJsonTest(#NAME, NAME, xpSettings); \ + } \ + \ + TEST_F(TYPE##JsonTest, NAME##_to_json) \ + { \ + writeJsonTest(#NAME, NAME); \ } TEST_JSON( @@ -156,7 +182,7 @@ TEST_JSON( .output = "bar", })); -TEST_JSON( +TEST_JSON_XP_DYN( SingleDerivedPath, single_built_built, (SingleDerivedPath::Built{ @@ -179,7 +205,7 @@ TEST_JSON( .outputs = OutputsSpec::Names{"bar", "baz"}, })); -TEST_JSON( +TEST_JSON_XP_DYN( DerivedPath, multi_built_built, (DerivedPath::Built{ @@ -191,7 +217,7 @@ TEST_JSON( .outputs = OutputsSpec::Names{"baz", "quux"}, })); -TEST_JSON( +TEST_JSON_XP_DYN( DerivedPath, multi_built_built_wildcard, (DerivedPath::Built{ diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 8d606cb41..251e11251 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -252,20 +252,26 @@ void adl_serializer::to_json(json & json, const DerivedPath: }; } -SingleDerivedPath::Built adl_serializer::from_json(const json & json0) +SingleDerivedPath::Built +adl_serializer::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings) { auto & json = getObject(json0); + auto drvPath = make_ref(static_cast(valueAt(json, "drvPath"))); + drvRequireExperiment(*drvPath, xpSettings); return { - .drvPath = make_ref(static_cast(valueAt(json, "drvPath"))), + .drvPath = std::move(drvPath), .output = getString(valueAt(json, "output")), }; } -DerivedPath::Built adl_serializer::from_json(const json & json0) +DerivedPath::Built +adl_serializer::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings) { auto & json = getObject(json0); + auto drvPath = make_ref(static_cast(valueAt(json, "drvPath"))); + drvRequireExperiment(*drvPath, xpSettings); return { - .drvPath = make_ref(static_cast(valueAt(json, "drvPath"))), + .drvPath = std::move(drvPath), .outputs = adl_serializer::from_json(valueAt(json, "outputs")), }; } @@ -280,20 +286,21 @@ void adl_serializer::to_json(json & json, const DerivedPath & sdp) std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); } -SingleDerivedPath adl_serializer::from_json(const json & json) +SingleDerivedPath +adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) { if (json.is_string()) return static_cast(json); else - return static_cast(json); + return adl_serializer::from_json(json, xpSettings); } -DerivedPath adl_serializer::from_json(const json & json) +DerivedPath adl_serializer::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings) { if (json.is_string()) return static_cast(json); else - return static_cast(json); + return adl_serializer::from_json(json, xpSettings); } } // namespace nlohmann diff --git a/src/libstore/include/nix/store/derived-path.hh b/src/libstore/include/nix/store/derived-path.hh index 47b29b2d6..70074ea40 100644 --- a/src/libstore/include/nix/store/derived-path.hh +++ b/src/libstore/include/nix/store/derived-path.hh @@ -299,7 +299,7 @@ void drvRequireExperiment( } // namespace nix JSON_IMPL(nix::SingleDerivedPath::Opaque) -JSON_IMPL(nix::SingleDerivedPath::Built) -JSON_IMPL(nix::SingleDerivedPath) -JSON_IMPL(nix::DerivedPath::Built) -JSON_IMPL(nix::DerivedPath) +JSON_IMPL_WITH_XP_FEATURES(nix::SingleDerivedPath::Built) +JSON_IMPL_WITH_XP_FEATURES(nix::SingleDerivedPath) +JSON_IMPL_WITH_XP_FEATURES(nix::DerivedPath::Built) +JSON_IMPL_WITH_XP_FEATURES(nix::DerivedPath) From 646d1f5ff7b6ee7e707f496b0d3c1cc0064bf6b7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Oct 2025 16:54:47 -0400 Subject: [PATCH 016/201] `registerOutputs`: Swap check and non-check cases in the code This is the exact same control flow, but with one less branch because it becomes redundant. --- src/libstore/unix/build/derivation-builder.cc | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 0efdc14b2..ae161ae8c 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1737,27 +1737,6 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); } - /* Move files, if needed */ - if (store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(store.toRealPath(finalDestPath), actualPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - /* Can delete our scratch copy now. */ - deletePath(actualPath); - } else { - auto destPath = store.toRealPath(finalDestPath); - deletePath(destPath); - movePath(actualPath, destPath); - } - } - if (buildMode == bmCheck) { /* Check against already registered outputs */ @@ -1799,6 +1778,24 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() } else { /* do tasks relating to registering these outputs */ + /* Move files, if needed */ + if (store.toRealPath(finalDestPath) != actualPath) { + if (buildMode == bmRepair) { + /* Path already exists, need to replace it */ + replaceValidPath(store.toRealPath(finalDestPath), actualPath); + } else if (store.isValidPath(newInfo.path)) { + /* Path already exists because CA path produced by something + else. No moving needed. */ + assert(newInfo.ca); + /* Can delete our scratch copy now. */ + deletePath(actualPath); + } else { + auto destPath = store.toRealPath(finalDestPath); + deletePath(destPath); + movePath(actualPath, destPath); + } + } + /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { if (references.count(i)) From 01b001d5ba710637a24ab1432533acdb7bc1292a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Oct 2025 15:32:23 +0200 Subject: [PATCH 017/201] Add JSON Schema infrastructure, use for Derivation For manual, and testing formats --- ci/gha/tests/default.nix | 1 + doc/manual/meson.build | 1 + doc/manual/package.nix | 2 + doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/meson.build | 3 + .../source/protocols/json/derivation.md | 123 +------------ .../json/fixup-json-schema-generated-doc.sed | 14 ++ doc/manual/source/protocols/json/hash.md | 7 + .../json/json-schema-for-humans-config.yaml | 17 ++ doc/manual/source/protocols/json/meson.build | 74 ++++++++ .../protocols/json/schema/derivation-v3.yaml | 164 ++++++++++++++++++ .../source/protocols/json/schema/hash-v1.yaml | 29 ++++ doc/manual/source/protocols/meson.build | 2 + flake.nix | 4 + meson.build | 1 + packaging/components.nix | 5 + packaging/dev-shell.nix | 1 + packaging/hydra.nix | 1 + src/json-schema-checks/.version | 1 + src/json-schema-checks/derivation | 1 + src/json-schema-checks/meson.build | 76 ++++++++ src/json-schema-checks/package.nix | 50 ++++++ src/json-schema-checks/schema | 1 + src/nix/derivation-add.md | 5 +- src/nix/derivation-show.md | 5 +- 25 files changed, 465 insertions(+), 124 deletions(-) create mode 100644 doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed create mode 100644 doc/manual/source/protocols/json/hash.md create mode 100644 doc/manual/source/protocols/json/json-schema-for-humans-config.yaml create mode 100644 doc/manual/source/protocols/json/meson.build create mode 100644 doc/manual/source/protocols/json/schema/derivation-v3.yaml create mode 100644 doc/manual/source/protocols/json/schema/hash-v1.yaml create mode 100644 doc/manual/source/protocols/meson.build create mode 120000 src/json-schema-checks/.version create mode 120000 src/json-schema-checks/derivation create mode 100644 src/json-schema-checks/meson.build create mode 100644 src/json-schema-checks/package.nix create mode 120000 src/json-schema-checks/schema diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 2bfdae17b..0c5c103bf 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -116,6 +116,7 @@ rec { ) nixComponentsInstrumented) // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) { "${componentTestsPrefix}nix-functional-tests" = nixComponentsInstrumented.nix-functional-tests; + "${componentTestsPrefix}nix-json-schema-checks" = nixComponentsInstrumented.nix-json-schema-checks; }; codeCoverage = diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 2e372dedd..7090c949c 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -115,6 +115,7 @@ manual = custom_target( builtins_md, rl_next_generated, summary_rl_next, + json_schema_generated_files, nix_input, ], output : [ diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 69b7c0e49..7b94721ae 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -12,6 +12,7 @@ rsync, nix-cli, changelog-d, + json-schema-for-humans, officialRelease, # Configuration Options @@ -55,6 +56,7 @@ mkMesonDerivation (finalAttrs: { jq python3 rsync + json-schema-for-humans changelog-d ] ++ lib.optionals (!officialRelease) [ diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 25e68811d..f74ed7043 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -117,6 +117,7 @@ - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) + - [Hash](protocols/json/hash.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) diff --git a/doc/manual/source/meson.build b/doc/manual/source/meson.build index 949d26526..294d57ad9 100644 --- a/doc/manual/source/meson.build +++ b/doc/manual/source/meson.build @@ -1,3 +1,6 @@ +# Process JSON schema documentation +subdir('protocols') + summary_rl_next = custom_target( command : [ bash, diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index cc9389f7c..602ab67e4 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,120 +1,7 @@ -# Derivation JSON Format +{{#include derivation-v3-fixed.md}} -> **Warning** -> -> This JSON format is currently -> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> and subject to change. + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed new file mode 100644 index 000000000..126e666e9 --- /dev/null +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -0,0 +1,14 @@ +# For some reason, backticks in the JSON schema are being escaped rather +# than being kept as intentional code spans. This removes all backtick +# escaping, which is an ugly solution, but one that is fine, because we +# are not using backticks for any other purpose. +s/\\`/`/g + +# The way that semi-external references are rendered (i.e. ones to +# sibling schema files, as opposed to separate website ones, is not nice +# for humans. Replace it with a nice relative link within the manual +# instead. +# +# As we have more such relative links, more replacements of this nature +# should appear below. +s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g diff --git a/doc/manual/source/protocols/json/hash.md b/doc/manual/source/protocols/json/hash.md new file mode 100644 index 000000000..d2bdf1062 --- /dev/null +++ b/doc/manual/source/protocols/json/hash.md @@ -0,0 +1,7 @@ +{{#include hash-v1-fixed.md}} + + diff --git a/doc/manual/source/protocols/json/json-schema-for-humans-config.yaml b/doc/manual/source/protocols/json/json-schema-for-humans-config.yaml new file mode 100644 index 000000000..cad098053 --- /dev/null +++ b/doc/manual/source/protocols/json/json-schema-for-humans-config.yaml @@ -0,0 +1,17 @@ +# Configuration file for json-schema-for-humans +# +# https://github.com/coveooss/json-schema-for-humans/blob/main/docs/examples/examples_md_default/Configuration.md + +template_name: md +show_toc: true +# impure timestamp and distracting +with_footer: false +recursive_detection_depth: 3 +show_breadcrumbs: false +description_is_markdown: true +template_md_options: + properties_table_columns: + - Property + - Type + - Pattern + - Title/Description diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build new file mode 100644 index 000000000..44795599c --- /dev/null +++ b/doc/manual/source/protocols/json/meson.build @@ -0,0 +1,74 @@ +# Tests in: ../../../../src/json-schema-checks + +fs = import('fs') + +# Find json-schema-for-humans if available +json_schema_for_humans = find_program('generate-schema-doc', required : false) + +# Configuration for json-schema-for-humans +json_schema_config = files('json-schema-for-humans-config.yaml') + +schemas = [ + 'hash-v1', + 'derivation-v3', +] + +schema_files = files() +foreach schema_name : schemas + schema_files += files('schema' / schema_name + '.yaml') +endforeach + + +schema_outputs = [] +foreach schema_name : schemas + schema_outputs += schema_name + '.md' +endforeach + +json_schema_generated_files = [] + +# Generate markdown documentation from JSON schema +# Note: output must be just a filename, not a path +gen_file = custom_target( + schema_name + '-schema-docs.tmp', + command : [ + json_schema_for_humans, + '--config-file', + json_schema_config, + meson.current_source_dir() / 'schema', + meson.current_build_dir(), + ], + input : schema_files + [ + json_schema_config, + ], + output : schema_outputs, + capture : false, + build_by_default : true, +) + +idx = 0 +if json_schema_for_humans.found() + foreach schema_name : schemas + #schema_file = 'schema' / schema_name + '.yaml' + + # There is one so-so hack, and one horrible hack being done here. + sedded_file = custom_target( + schema_name + '-schema-docs', + command : [ + 'sed', + '-f', + # Out of line to avoid https://github.com/mesonbuild/meson/issues/1564 + files('fixup-json-schema-generated-doc.sed'), + '@INPUT@', + ], + capture : true, + input : gen_file[idx], + output : schema_name + '-fixed.md', + ) + idx += 1 + json_schema_generated_files += [ sedded_file ] + endforeach +else + warning( + 'json-schema-for-humans not found, skipping JSON schema documentation generation', + ) +endif diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml new file mode 100644 index 000000000..e80f24e9f --- /dev/null +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -0,0 +1,164 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json +title: Derivation +description: | + Experimental JSON representation of a Nix derivation (version 3). + + This schema describes the JSON representation of Nix's `Derivation` type. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + +type: object +required: + - name + - version + - outputs + - inputSrcs + - inputDrvs + - system + - builder + - args + - env +properties: + name: + type: string + description: | + The name of the derivation. + Used when calculating store paths for the derivation’s outputs. + + version: + const: 3 + description: | + Must be `3`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: + + - Version 0: A-Term format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format. + + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Version 3: Drop store dir from store paths, just include base name. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + + outputs: + type: object + description: | + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "method": "nar", + > "hashAlgo": "sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + additionalProperties: + "$ref": "#/$defs/output" + + inputSrcs: + type: array + description: | + List of store paths on which this derivation depends. + + > **Example** + > + > ```json + > "inputSrcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + items: + type: string + + inputDrvs: + type: object + description: | + Mapping of derivation paths to lists of output names they provide. + + > **Example** + > + > ```json + > "inputDrvs": { + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > } + > ``` + > + > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + + system: + type: string + description: | + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + + builder: + type: string + description: | + Absolute path of the program used to perform the build. + Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + + args: + type: array + description: | + Command-line arguments passed to the `builder`. + items: + type: string + + env: + type: object + description: | + Environment variables passed to the `builder`. + additionalProperties: + type: string + + structuredAttrs: + description: | + [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. + Structured attributes are JSON, and thus embedded as-is. + type: object + additionalProperties: true + +"$defs": + output: + type: object + properties: + path: + type: string + description: | + The output path, if known in advance. + + method: + type: string + enum: [flat, nar, text, git] + description: | + For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + hashAlgo: + "$ref": "./hash-v1.yaml#/$defs/algorithm" + + hash: + type: string + description: | + For fixed-output derivations, the expected content hash in base-16. diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml new file mode 100644 index 000000000..b258a90c7 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -0,0 +1,29 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json +title: Hash +description: | + A cryptographic hash value used throughout Nix for content addressing and integrity verification. + + This schema describes the JSON representation of Nix's `Hash` type. + + TODO Work in progress +type: object +properties: + algorithm: + "$ref": "#/$defs/algorithm" +required: +- algorithm +additionalProperties: false +"$defs": + algorithm: + type: string + enum: + - blake3 + - md5 + - sha1 + - sha256 + - sha512 + description: | + The hash algorithm used to compute the hash value. + + `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake-hashing) experimental feature. diff --git a/doc/manual/source/protocols/meson.build b/doc/manual/source/protocols/meson.build new file mode 100644 index 000000000..5b5eb900d --- /dev/null +++ b/doc/manual/source/protocols/meson.build @@ -0,0 +1,2 @@ +# Process JSON schema documentation +subdir('json') diff --git a/flake.nix b/flake.nix index fd623c807..8d3d963be 100644 --- a/flake.nix +++ b/flake.nix @@ -413,6 +413,10 @@ supportsCross = false; }; + "nix-json-schema-checks" = { + supportsCross = false; + }; + "nix-perl-bindings" = { supportsCross = false; }; diff --git a/meson.build b/meson.build index 736756157..f3158ea6d 100644 --- a/meson.build +++ b/meson.build @@ -60,3 +60,4 @@ if get_option('unit-tests') subproject('libflake-tests') endif subproject('nix-functional-tests') +subproject('json-schema-checks') diff --git a/packaging/components.nix b/packaging/components.nix index c621b7073..f9d7b109a 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -438,6 +438,11 @@ in */ nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; + /** + JSON schema validation checks + */ + nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; + nix-perl-bindings = callPackage ../src/perl/package.nix { }; /** diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 5fb4f14d2..153e7a3eb 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -108,6 +108,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs + ++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs ++ lib.optional ( !buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 diff --git a/packaging/hydra.nix b/packaging/hydra.nix index bc75b5dfb..3bbb6c15b 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -62,6 +62,7 @@ let "nix-cmd" "nix-cli" "nix-functional-tests" + "nix-json-schema-checks" ] ++ lib.optionals enableBindings [ "nix-perl-bindings" diff --git a/src/json-schema-checks/.version b/src/json-schema-checks/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/json-schema-checks/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/json-schema-checks/derivation b/src/json-schema-checks/derivation new file mode 120000 index 000000000..3dc1cbe06 --- /dev/null +++ b/src/json-schema-checks/derivation @@ -0,0 +1 @@ +../../src/libstore-tests/data/derivation \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build new file mode 100644 index 000000000..5326ad4c6 --- /dev/null +++ b/src/json-schema-checks/meson.build @@ -0,0 +1,76 @@ +# Run with: +# meson test --suite json-schema +# Run with: (without shell / configure) +# nix build .#nix-json-schema-checks + +project( + 'nix-json-schema-checks', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +# Note: The 'jsonschema' package provides the 'jv' command +jv = find_program('jv', required : true) + +# The schema directory is a committed symlink to the actual schema location +schema_dir = meson.current_source_dir() / 'schema' + +# Get all example files +schemas = [ + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-v3.yaml', + 'files' : [ + 'dyn-dep-derivation.json', + 'simple-derivation.json', + ], + }, + # # Not sure how to make subschema work + # { + # 'stem': 'derivation', + # 'schema': schema_dir / 'derivation-v3.yaml#output', + # 'files' : [ + # 'output-caFixedFlat.json', + # 'output-caFixedNAR.json', + # 'output-caFixedText.json', + # 'output-caFloating.json', + # 'output-deferred.json', + # 'output-impure.json', + # 'output-inputAddressed.json', + # ], + # }, +] + +# Validate each example against the schema +foreach schema : schemas + stem = schema['stem'] + schema_file = schema['schema'] + if '#' not in schema_file + # Validate the schema itself against JSON Schema Draft 04 + test( + stem + '-schema-valid', + jv, + args : [ + '--map', + './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml', + 'http://json-schema.org/draft-04/schema', + schema_file, + ], + suite : 'json-schema', + ) + endif + foreach example : schema['files'] + test( + stem + '-example-' + fs.stem(example), + jv, + args : [ + schema_file, + files(stem / example), + ], + suite : 'json-schema', + ) + endforeach +endforeach diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix new file mode 100644 index 000000000..2061672cd --- /dev/null +++ b/src/json-schema-checks/package.nix @@ -0,0 +1,50 @@ +# Run with: nix build .#nix-json-schema-checks +{ + lib, + mkMesonDerivation, + + meson, + ninja, + jsonschema, + + # Configuration Options + + version, +}: + +mkMesonDerivation (finalAttrs: { + pname = "nix-json-schema-checks"; + inherit version; + + workDir = ./.; + fileset = lib.fileset.unions [ + ../../.version + ../../doc/manual/source/protocols/json/schema + ../../src/libstore-tests/data/derivation + ./. + ]; + + outputs = [ "out" ]; + + passthru.externalNativeBuildInputs = [ + jsonschema + ]; + + nativeBuildInputs = [ + meson + ninja + ] + ++ finalAttrs.passthru.externalNativeBuildInputs; + + doCheck = true; + + mesonCheckFlags = [ "--print-errorlogs" ]; + + postInstall = '' + touch $out + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/json-schema-checks/schema b/src/json-schema-checks/schema new file mode 120000 index 000000000..473e47b1b --- /dev/null +++ b/src/json-schema-checks/schema @@ -0,0 +1 @@ +../../doc/manual/source/protocols/json/schema \ No newline at end of file diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index 35507d9ad..4e37c4e6f 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -12,8 +12,7 @@ a Nix expression evaluates. [store derivation]: @docroot@/glossary.md#gloss-store-derivation -`nix derivation add` takes a single derivation in the following format: - -{{#include ../../protocols/json/derivation.md}} +`nix derivation add` takes a single derivation in the JSON format. +See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format. )"" diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 9fff58ef9..1784be44c 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -48,10 +48,9 @@ By default, this command only shows top-level derivations, but with [store derivation]: @docroot@/glossary.md#gloss-store-derivation -`nix derivation show` outputs a JSON map of [store path]s to derivations in the following format: +`nix derivation show` outputs a JSON map of [store path]s to derivations in JSON format. +See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format. [store path]: @docroot@/store/store-path.md -{{#include ../../protocols/json/derivation.md}} - )"" From bcd5a9d05ce6faf8da520e8423ad832c332eed35 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Oct 2025 00:56:53 +0300 Subject: [PATCH 018/201] libutil: Drop unused SubdirSourceAccessor --- .../include/nix/util/source-accessor.hh | 6 -- src/libutil/meson.build | 1 - src/libutil/subdir-source-accessor.cc | 59 ------------------- 3 files changed, 66 deletions(-) delete mode 100644 src/libutil/subdir-source-accessor.cc diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 671444e6f..1006895b3 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -241,10 +241,4 @@ ref makeFSSourceAccessor(std::filesystem::path root); */ ref makeUnionSourceAccessor(std::vector> && accessors); -/** - * Creates a new source accessor which is confined to the subdirectory - * of the given source accessor. - */ -ref projectSubdirSourceAccessor(ref, CanonPath subdirectory); - } // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index f4b8dbb61..acba0b81b 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -156,7 +156,6 @@ sources = [ config_priv_h ] + files( 'source-accessor.cc', 'source-path.cc', 'strings.cc', - 'subdir-source-accessor.cc', 'suggestions.cc', 'tarfile.cc', 'tee-logger.cc', diff --git a/src/libutil/subdir-source-accessor.cc b/src/libutil/subdir-source-accessor.cc deleted file mode 100644 index d4f57e2f7..000000000 --- a/src/libutil/subdir-source-accessor.cc +++ /dev/null @@ -1,59 +0,0 @@ -#include "nix/util/source-accessor.hh" - -namespace nix { - -struct SubdirSourceAccessor : SourceAccessor -{ - ref parent; - - CanonPath subdirectory; - - SubdirSourceAccessor(ref && parent, CanonPath && subdirectory) - : parent(std::move(parent)) - , subdirectory(std::move(subdirectory)) - { - displayPrefix.clear(); - } - - std::string readFile(const CanonPath & path) override - { - return parent->readFile(subdirectory / path); - } - - void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override - { - return parent->readFile(subdirectory / path, sink, sizeCallback); - } - - bool pathExists(const CanonPath & path) override - { - return parent->pathExists(subdirectory / path); - } - - std::optional maybeLstat(const CanonPath & path) override - { - return parent->maybeLstat(subdirectory / path); - } - - DirEntries readDirectory(const CanonPath & path) override - { - return parent->readDirectory(subdirectory / path); - } - - std::string readLink(const CanonPath & path) override - { - return parent->readLink(subdirectory / path); - } - - std::string showPath(const CanonPath & path) override - { - return displayPrefix + parent->showPath(subdirectory / path) + displaySuffix; - } -}; - -ref projectSubdirSourceAccessor(ref parent, CanonPath subdirectory) -{ - return make_ref(std::move(parent), std::move(subdirectory)); -} - -} // namespace nix From a80fc252e8dc56e991322f0871490ef264071c3e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Oct 2025 01:10:46 +0300 Subject: [PATCH 019/201] libstore/meson: Require curl >= 7.75.0 This version has been released a long time ago in 2021 and it's doubtful that anybody actually uses it still, since it's full of vulnerabilities [^] [^]: https://curl.se/docs/vuln-7.75.0.html --- src/libstore/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 40da06e6b..a0061eb9b 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -113,7 +113,7 @@ boost = dependency( # put in `deps_other`. deps_other += boost -curl = dependency('libcurl', 'curl') +curl = dependency('libcurl', 'curl', version : '>= 7.75.0') deps_private += curl # seccomp only makes sense on Linux From ffbc33fec610fe97a795edc2bee481252dbc902b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Oct 2025 01:18:46 +0300 Subject: [PATCH 020/201] libstore/meson: Rename curl-s3-store to s3-aws-auth We now unconditionally compile support for s3:// URLs and stores without authentication. The whole curl version check can be greatly simplified by the previous commit, which bumps the minimum required curl version. --- src/libstore/meson.build | 21 +++++---------------- src/libstore/meson.options | 5 ++--- src/libstore/package.nix | 2 +- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index a0061eb9b..d1b3666cc 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -142,27 +142,16 @@ deps_public += nlohmann_json sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite -# Curl-based S3 store support -# Check if curl supports AWS SigV4 (requires >= 7.75.0) -curl_supports_aws_sigv4 = curl.version().version_compare('>= 7.75.0') -# AWS CRT C++ for lightweight credential management -aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : false) +s3_aws_auth = get_option('s3-aws-auth') +aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : s3_aws_auth) -curl_s3_store_opt = get_option('curl-s3-store').require( - curl_supports_aws_sigv4, - error_message : 'curl-based S3 support requires curl >= 7.75.0', -).require( - aws_crt_cpp.found(), - error_message : 'curl-based S3 support requires aws-crt-cpp', -) - -if curl_s3_store_opt.enabled() +if s3_aws_auth.enabled() deps_other += aws_crt_cpp aws_c_common = cxx.find_library('aws-c-common', required : true) deps_other += aws_c_common endif -configdata_pub.set('NIX_WITH_AWS_AUTH', curl_s3_store_opt.enabled().to_int()) +configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int()) subdir('nix-meson-build-support/generate-header') @@ -346,7 +335,7 @@ sources = files( ) # AWS credentials code requires AWS CRT, so only compile when enabled -if curl_s3_store_opt.enabled() +if s3_aws_auth.enabled() sources += files('aws-creds.cc') endif diff --git a/src/libstore/meson.options b/src/libstore/meson.options index edc43bd45..c822133df 100644 --- a/src/libstore/meson.options +++ b/src/libstore/meson.options @@ -35,8 +35,7 @@ option( ) option( - 'curl-s3-store', + 's3-aws-auth', type : 'feature', - value : 'disabled', - description : 'Enable curl-based S3 binary cache store support (requires aws-crt-cpp and curl >= 7.75.0)', + description : 'build support for AWS authentication with S3', ) diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 897662e11..ddad077ce 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -75,7 +75,7 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux) (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell) - (lib.mesonEnable "curl-s3-store" withAWS) + (lib.mesonEnable "s3-aws-auth" withAWS) ] ++ lib.optionals stdenv.hostPlatform.isLinux [ (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") From 64c55961eb52d298e0643481c31fd178a59f5cd4 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Thu, 16 Oct 2025 16:16:54 -0700 Subject: [PATCH 021/201] Merge pull request #14273 from fzakaria/fzakaria/issue-13944 Make `nix nar [cat|ls]` lazy --- .../include/nix/store/nar-accessor.hh | 7 +++- src/libstore/nar-accessor.cc | 34 +++++++++++++++---- src/libstore/remote-fs-accessor.cc | 22 ++---------- src/nix/cat.cc | 10 +++++- src/nix/ls.cc | 6 +++- 5 files changed, 50 insertions(+), 29 deletions(-) diff --git a/src/libstore/include/nix/store/nar-accessor.hh b/src/libstore/include/nix/store/nar-accessor.hh index 0e69d436e..bfba5da73 100644 --- a/src/libstore/include/nix/store/nar-accessor.hh +++ b/src/libstore/include/nix/store/nar-accessor.hh @@ -27,7 +27,12 @@ ref makeNarAccessor(Source & source); */ using GetNarBytes = std::function; -ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes); +/** + * The canonical GetNarBytes function for a seekable Source. + */ +GetNarBytes seekableGetNarBytes(const Path & path); + +ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes); /** * Write a JSON representation of the contents of a NAR (except file diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 63fe774c9..f0882d52d 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -141,14 +141,14 @@ struct NarAccessor : public SourceAccessor parseDump(indexer, indexer); } - NarAccessor(const std::string & listing, GetNarBytes getNarBytes) + NarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes) : getNarBytes(getNarBytes) { using json = nlohmann::json; - std::function recurse; + std::function recurse; - recurse = [&](NarMember & member, json & v) { + recurse = [&](NarMember & member, const json & v) { std::string type = v["type"]; if (type == "directory") { @@ -169,8 +169,7 @@ struct NarAccessor : public SourceAccessor return; }; - json v = json::parse(listing); - recurse(root, v); + recurse(root, listing); } NarMember * find(const CanonPath & path) @@ -251,11 +250,34 @@ ref makeNarAccessor(Source & source) return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) +ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } +GetNarBytes seekableGetNarBytes(const Path & path) +{ + return [path](uint64_t offset, uint64_t length) { + AutoCloseFD fd = toDescriptor(open( + path.c_str(), + O_RDONLY +#ifndef _WIN32 + | O_CLOEXEC +#endif + )); + if (!fd) + throw SysError("opening NAR cache file '%s'", path); + + if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) + throw SysError("seeking in '%s'", path); + + std::string buf(length, 0); + readFull(fd.get(), buf.data(), length); + + return buf; + }; +} + using nlohmann::json; json listNar(ref accessor, const CanonPath & path, bool recurse) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index e6715cbdf..f7ca28ae2 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -70,26 +70,8 @@ std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); - - auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { - AutoCloseFD fd = toDescriptor(open( - cacheFile.c_str(), - O_RDONLY -#ifndef _WIN32 - | O_CLOEXEC -#endif - )); - if (!fd) - throw SysError("opening NAR cache file '%s'", cacheFile); - - if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) - throw SysError("seeking in '%s'", cacheFile); - - std::string buf(length, 0); - readFull(fd.get(), buf.data(), length); - - return buf; - }); + auto listingJson = nlohmann::json::parse(listing); + auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile)); nars.emplace(storePath.hashPart(), narAccessor); return narAccessor; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index effe544e6..5b93d560b 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,10 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" #include "nix/store/nar-accessor.hh" +#include "nix/util/serialise.hh" +#include "nix/util/source-accessor.hh" + +#include using namespace nix; @@ -71,7 +75,11 @@ struct CmdCatNar : StoreCommand, MixCat void run(ref store) override { - cat(makeNarAccessor(readFile(narPath)), CanonPath{path}); + AutoCloseFD fd = open(narPath.c_str(), O_RDONLY); + auto source = FdSource{fd.get()}; + auto narAccessor = makeNarAccessor(source); + auto listing = listNar(narAccessor, CanonPath::root, true); + cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 5cdfc2c0f..846af246d 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -145,7 +145,11 @@ struct CmdLsNar : Command, MixLs void run() override { - list(makeNarAccessor(readFile(narPath)), CanonPath{path}); + AutoCloseFD fd = open(narPath.c_str(), O_RDONLY); + auto source = FdSource{fd.get()}; + auto narAccessor = makeNarAccessor(source); + auto listing = listNar(narAccessor, CanonPath::root, true); + list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; From e457ea768880149b259429c836b2260f0a2c6b8f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Oct 2025 02:26:24 +0300 Subject: [PATCH 022/201] nix {cat,ls}: Add back missing checks for file descriptors I didn't catch this during the review of https://github.com/NixOS/nix/pull/14273. This fixes that mistake. --- src/nix/cat.cc | 2 ++ src/nix/ls.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 5b93d560b..1284b50fd 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -76,6 +76,8 @@ struct CmdCatNar : StoreCommand, MixCat void run(ref store) override { AutoCloseFD fd = open(narPath.c_str(), O_RDONLY); + if (!fd) + throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); auto listing = listNar(narAccessor, CanonPath::root, true); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 846af246d..82721222e 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -146,6 +146,8 @@ struct CmdLsNar : Command, MixLs void run() override { AutoCloseFD fd = open(narPath.c_str(), O_RDONLY); + if (!fd) + throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); auto listing = listNar(narAccessor, CanonPath::root, true); From 35b08b71a4f04d6173401c8ba26426e84e54f827 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Oct 2025 16:59:19 -0400 Subject: [PATCH 023/201] `registerOutputs`: Hoist up `optimizePath` call and comment rationale --- src/libstore/unix/build/derivation-builder.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index ae161ae8c..9685f2054 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1783,16 +1783,26 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() if (buildMode == bmRepair) { /* Path already exists, need to replace it */ replaceValidPath(store.toRealPath(finalDestPath), actualPath); + /* Optimize store object we just replaced with new + (not-yet-optimized) data. */ + store.optimisePath( + store.toRealPath(finalDestPath), NoRepair); // FIXME: combine with scanForReferences() } else if (store.isValidPath(newInfo.path)) { /* Path already exists because CA path produced by something else. No moving needed. */ assert(newInfo.ca); /* Can delete our scratch copy now. */ deletePath(actualPath); + /* Presume already-existing store object is already + optimized. */ } else { auto destPath = store.toRealPath(finalDestPath); deletePath(destPath); movePath(actualPath, destPath); + /* Optimize store object we just installed from new + (not-yet-optimized) data. */ + store.optimisePath( + store.toRealPath(finalDestPath), NoRepair); // FIXME: combine with scanForReferences() } } @@ -1804,10 +1814,6 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() debug("unreferenced input: '%1%'", store.printStorePath(i)); } - if (!store.isValidPath(newInfo.path)) - store.optimisePath( - store.toRealPath(finalDestPath), NoRepair); // FIXME: combine with scanForReferences() - newInfo.deriver = drvPath; newInfo.ultimate = true; store.signPathInfo(newInfo); From 20c7c551bfdd5d99c477c4c04b4f5271bec4e285 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 17 Oct 2025 16:42:37 +0000 Subject: [PATCH 024/201] fix(tests/functional/repl): skip test if stack size limit is insufficient Nix attempts to set the stack size to 64 MB during initialization, which is required for the repl tests to run successfully. Skip the tests on systems where the hard stack limit is less than this value rather than failing. --- tests/functional/repl.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index bfe18c9e5..aeff43d30 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -25,6 +25,13 @@ import $testDir/undefined-variable.nix TODO_NixOS +# FIXME: repl tests fail on systems with stack limits +stack_ulimit="$(ulimit -Hs)" +stack_required="$((64 * 1024 * 1024))" +if [[ "$stack_ulimit" != "unlimited" ]]; then + ((stack_ulimit < stack_required)) && skipTest "repl tests cannot run on systems with stack size <$stack_required ($stack_ulimit)" +fi + testRepl () { local nixArgs nixArgs=("$@") From 109f6449cc782efae84f71bc68fe5ead1c752e3e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 20:23:20 +0200 Subject: [PATCH 025/201] nix store dump-path: Refuse to write NARs to the terminal --- src/nix/dump-path.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 8475655e9..f375b0ac8 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -4,6 +4,14 @@ using namespace nix; +static FdSink getNarSink() +{ + auto fd = getStandardOutput(); + if (isatty(fd)) + throw UsageError("refusing to write NAR to a terminal"); + return FdSink(std::move(fd)); +} + struct CmdDumpPath : StorePathCommand { std::string description() override @@ -20,7 +28,7 @@ struct CmdDumpPath : StorePathCommand void run(ref store, const StorePath & storePath) override { - FdSink sink(getStandardOutput()); + auto sink = getNarSink(); store->narFromPath(storePath, sink); sink.flush(); } @@ -51,7 +59,7 @@ struct CmdDumpPath2 : Command void run() override { - FdSink sink(getStandardOutput()); + auto sink = getNarSink(); dumpPath(path, sink); sink.flush(); } From daa7e0d2e967f455cbf6285f31c5307be49545cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 13:56:17 +0200 Subject: [PATCH 026/201] Source: Add skip() method This allows FdSource to efficiently skip data we don't care about. --- src/libutil/include/nix/util/serialise.hh | 5 +++ src/libutil/serialise.cc | 48 +++++++++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 16e0d0fa5..8799e128f 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -97,6 +97,8 @@ struct Source void drainInto(Sink & sink); std::string drain(); + + virtual void skip(size_t len); }; /** @@ -177,6 +179,7 @@ struct FdSource : BufferedSource Descriptor fd; size_t read = 0; BackedStringView endOfFileError{"unexpected end-of-file"}; + bool isSeekable = true; FdSource() : fd(INVALID_DESCRIPTOR) @@ -200,6 +203,8 @@ struct FdSource : BufferedSource */ bool hasData(); + void skip(size_t len) override; + protected: size_t readUnbuffered(char * data, size_t len) override; private: diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 15629935e..bdce956f3 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -94,9 +94,8 @@ void Source::drainInto(Sink & sink) { std::array buf; while (true) { - size_t n; try { - n = read(buf.data(), buf.size()); + auto n = read(buf.data(), buf.size()); sink({buf.data(), n}); } catch (EndOfFile &) { break; @@ -111,6 +110,16 @@ std::string Source::drain() return std::move(s.s); } +void Source::skip(size_t len) +{ + std::array buf; + while (len) { + auto n = read(buf.data(), std::min(len, buf.size())); + assert(n <= len); + len -= n; + } +} + size_t BufferedSource::read(char * data, size_t len) { if (!buffer) @@ -120,7 +129,7 @@ size_t BufferedSource::read(char * data, size_t len) bufPosIn = readUnbuffered(buffer.get(), bufSize); /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + auto n = std::min(len, bufPosIn - bufPosOut); memcpy(data, buffer.get() + bufPosOut, n); bufPosOut += n; if (bufPosIn == bufPosOut) @@ -191,6 +200,39 @@ bool FdSource::hasData() } } +void FdSource::skip(size_t len) +{ + /* Discard data in the buffer. */ + if (len && buffer && bufPosIn - bufPosOut) { + if (len >= bufPosIn - bufPosOut) { + len -= bufPosIn - bufPosOut; + bufPosIn = bufPosOut = 0; + } else { + bufPosOut += len; + len = 0; + } + } + +#ifndef _WIN32 + /* If we can, seek forward in the file to skip the rest. */ + if (isSeekable && len) { + if (lseek(fd, len, SEEK_CUR) == -1) { + if (errno == ESPIPE) + isSeekable = false; + else + throw SysError("seeking forward in file"); + } else { + read += len; + return; + } + } +#endif + + /* Otherwise, skip by reading. */ + if (len) + BufferedSource::skip(len); +} + size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) From 67bffa19a533c5d2b562db367d1823166ca714b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 18:32:47 +0200 Subject: [PATCH 027/201] NullFileSystemObjectSink: Skip over file contents --- src/libutil/archive.cc | 7 ++++++- src/libutil/fs-sink.cc | 2 ++ src/libutil/include/nix/util/fs-sink.hh | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3d96df75e..b8fef9ef3 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -132,6 +132,11 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) sink.preallocateContents(size); + if (sink.skipContents) { + source.skip(size + (size % 8 ? 8 - (size % 8) : 0)); + return; + } + uint64_t left = size; std::array buf; @@ -166,7 +171,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath auto expectTag = [&](std::string_view expected) { auto tag = getString(); if (tag != expected) - throw badArchive("expected tag '%s', got '%s'", expected, tag); + throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024)); }; expectTag("("); diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 6efd5e0c7..45ef57a9f 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -196,6 +196,8 @@ void NullFileSystemObjectSink::createRegularFile( void isExecutable() override {} } crf; + crf.skipContents = true; + // Even though `NullFileSystemObjectSink` doesn't do anything, it's important // that we call the function, to e.g. advance the parser using this // sink. diff --git a/src/libutil/include/nix/util/fs-sink.hh b/src/libutil/include/nix/util/fs-sink.hh index f96fe3ef9..bd2db7f53 100644 --- a/src/libutil/include/nix/util/fs-sink.hh +++ b/src/libutil/include/nix/util/fs-sink.hh @@ -14,6 +14,14 @@ namespace nix { */ struct CreateRegularFileSink : Sink { + /** + * If set to true, the sink will not be called with the contents + * of the file. `preallocateContents()` will still be called to + * convey the file size. Useful for sinks that want to efficiently + * discard the contents of the file. + */ + bool skipContents = false; + virtual void isExecutable() = 0; /** From c92ba4b9b7b63f24050c8e769a102380ea079e4f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 17 Oct 2025 21:50:31 +0200 Subject: [PATCH 028/201] Add titles in JSON schemas This way, the description isn't rendered in the tables of contents, leading to no more formatting errors. --- .../protocols/json/schema/derivation-v3.yaml | 14 ++++++++++++++ .../source/protocols/json/schema/hash-v1.yaml | 1 + 2 files changed, 15 insertions(+) diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index e80f24e9f..7c92d475d 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -26,12 +26,14 @@ required: properties: name: type: string + title: Derivation name description: | The name of the derivation. Used when calculating store paths for the derivation’s outputs. version: const: 3 + title: Format version (must be 3) description: | Must be `3`. This is a guard that allows us to continue evolving this format. @@ -49,6 +51,7 @@ properties: outputs: type: object + title: Output specifications description: | Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. @@ -69,6 +72,7 @@ properties: inputSrcs: type: array + title: Input source paths description: | List of store paths on which this derivation depends. @@ -85,6 +89,7 @@ properties: inputDrvs: type: object + title: Input derivations description: | Mapping of derivation paths to lists of output names they provide. @@ -101,12 +106,14 @@ properties: system: type: string + title: Build system type description: | The system type on which this derivation is to be built (e.g. `x86_64-linux`). builder: type: string + title: Build program path description: | Absolute path of the program used to perform the build. Typically this is the `bash` shell @@ -114,6 +121,7 @@ properties: args: type: array + title: Builder arguments description: | Command-line arguments passed to the `builder`. items: @@ -121,12 +129,14 @@ properties: env: type: object + title: Environment variables description: | Environment variables passed to the `builder`. additionalProperties: type: string structuredAttrs: + title: Structured attributes description: | [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. Structured attributes are JSON, and thus embedded as-is. @@ -139,11 +149,13 @@ properties: properties: path: type: string + title: Output path description: | The output path, if known in advance. method: type: string + title: Content addressing method enum: [flat, nar, text, git] description: | For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. @@ -156,9 +168,11 @@ properties: - [`git`](@docroot@/store/store-object/content-address.md#method-git) hashAlgo: + title: Hash algorithm "$ref": "./hash-v1.yaml#/$defs/algorithm" hash: type: string + title: Expected hash value description: | For fixed-output derivations, the expected content hash in base-16. diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml index b258a90c7..44a59541b 100644 --- a/doc/manual/source/protocols/json/schema/hash-v1.yaml +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -10,6 +10,7 @@ description: | type: object properties: algorithm: + title: Hash algorithm "$ref": "#/$defs/algorithm" required: - algorithm From 61fbef42a6eeae7553f148f1759c5a770a2f65aa Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 18 Oct 2025 18:47:27 +0300 Subject: [PATCH 029/201] libstore: Simplify check for S3-specific URI query parameters Instead of hardcoding strings we should instead use the setting objects to determine the query names that should be preserved. --- .../include/nix/store/s3-binary-cache-store.hh | 8 ++++++-- src/libstore/s3-binary-cache-store.cc | 15 +++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index c8cb967c1..e5fcbeda3 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -21,8 +21,6 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig Nix uses the `default` profile. )"}; -public: - const Setting region{ this, "us-east-1", @@ -63,6 +61,12 @@ public: > addressing instead of virtual host based addressing. )"}; + /** + * Set of settings that are part of the S3 URI itself. + * These are needed for region specification and other S3-specific settings. + */ + const std::set s3UriSettings = {&profile, ®ion, &scheme, &endpoint}; + static const std::string name() { return "S3 Binary Cache Store"; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index a84ea5fcb..ac08a4982 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,10 +1,10 @@ #include "nix/store/s3-binary-cache-store.hh" - -#include - #include "nix/store/http-binary-cache-store.hh" #include "nix/store/store-registration.hh" +#include +#include + namespace nix { StringSet S3BinaryCacheStoreConfig::uriSchemes() @@ -17,14 +17,13 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( : StoreConfig(params) , HttpBinaryCacheStoreConfig(scheme, _cacheUri, params) { - // For S3 stores, preserve S3-specific query parameters as part of the URL - // These are needed for region specification and other S3-specific settings assert(cacheUri.query.empty()); + assert(cacheUri.scheme == "s3"); - // Only copy S3-specific parameters to the URL query - static const std::set s3Params = {"region", "endpoint", "profile", "scheme"}; for (const auto & [key, value] : params) { - if (s3Params.contains(key)) { + auto s3Params = + std::views::transform(s3UriSettings, [](const AbstractSetting * setting) { return setting->name; }); + if (std::ranges::contains(s3Params, key)) { cacheUri.query[key] = value; } } From 3d147c04a5f9d03e1696fb25b495a077885d2cf7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 18 Oct 2025 19:11:39 +0300 Subject: [PATCH 030/201] libstore: Implement getHumanReadableURI for S3BinaryCacheStoreConfig This slightly improves the logs situation by including the region/profile/endpoint in the logs when S3 store references get printed. Instead of: copying path '/nix/store/lxnp9cs4cfh2g9r2bs4z7gwwz9kdj2r9-test-package-c' to 's3://bucketname'... This now includes: copying path '/nix/store/lxnp9cs4cfh2g9r2bs4z7gwwz9kdj2r9-test-package-c' to 's3://bucketname?endpoint=http://server:9000®ion=eu-west-1'... --- .../include/nix/store/s3-binary-cache-store.hh | 2 ++ src/libstore/s3-binary-cache-store.cc | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index e5fcbeda3..288ca41a0 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -75,6 +75,8 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig static StringSet uriSchemes(); static std::string doc(); + + std::string getHumanReadableURI() const override; }; } // namespace nix diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ac08a4982..0b37ac5d7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -29,6 +29,19 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( } } +std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const +{ + auto reference = getReference(); + reference.params = [&]() { + Params relevantParams; + for (auto & setting : s3UriSettings) + if (setting->overridden) + relevantParams.insert({setting->name, reference.params.at(setting->name)}); + return relevantParams; + }(); + return reference.render(); +} + std::string S3BinaryCacheStoreConfig::doc() { return R"( From 22f4cccc716abbb2ce58622bed699d3259bdd724 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:15:53 +0000 Subject: [PATCH 031/201] refactor(tests/nixos/s3-binary-cache-store): use a PKGS dict Replace individual PKG_A, PKG_B, and PKG_C variables with a PKGS dictionary. This will enable `@with_clean_client_store` in the future. --- tests/nixos/s3-binary-cache-store.nix | 46 ++++++++++++++------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 53d79689c..2d5c6c1c1 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -83,9 +83,11 @@ in ENDPOINT = 'http://server:9000' REGION = 'eu-west-1' - PKG_A = '${pkgA}' - PKG_B = '${pkgB}' - PKG_C = '${pkgC}' + PKGS = { + 'A': '${pkgA}', + 'B': '${pkgB}', + 'C': '${pkgC}', + } ENV_WITH_CREDS = f"AWS_ACCESS_KEY_ID={ACCESS_KEY} AWS_SECRET_ACCESS_KEY={SECRET_KEY}" @@ -168,7 +170,7 @@ in store_url = make_s3_url(bucket) output = server.succeed( f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' " - f"{PKG_A} {PKG_B} {PKG_C} 2>&1" + f"{PKGS['A']} {PKGS['B']} {PKGS['C']} 2>&1" ) assert_count( @@ -180,7 +182,7 @@ in print("✓ Credential provider created once and cached") - @with_test_bucket(populate_with=[PKG_A]) + @with_test_bucket(populate_with=[PKGS['A']]) def test_fetchurl_basic(bucket): """Test builtins.fetchurl works with s3:// URLs""" print("\n=== Testing builtins.fetchurl ===") @@ -216,7 +218,7 @@ in print("✓ Error messages format URLs correctly") - @with_test_bucket(populate_with=[PKG_A]) + @with_test_bucket(populate_with=[PKGS['A']]) def test_fork_credential_preresolution(bucket): """Test credential pre-resolution in forked processes""" print("\n=== Testing Fork Credential Pre-resolution ===") @@ -296,7 +298,7 @@ in print(" ✓ Child uses pre-resolved credentials (no new providers)") - @with_test_bucket(populate_with=[PKG_A, PKG_B, PKG_C]) + @with_test_bucket(populate_with=[PKGS['A'], PKGS['B'], PKGS['C']]) def test_store_operations(bucket): """Test nix store info and copy operations""" print("\n=== Testing Store Operations ===") @@ -316,11 +318,11 @@ in print(f" ✓ Store URL: {store_info['url']}") # Test copy from store - client.fail(f"nix path-info {PKG_A}") + client.fail(f"nix path-info {PKGS['A']}") output = client.succeed( f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs " - f"--from '{store_url}' {PKG_A} {PKG_B} {PKG_C} 2>&1" + f"--from '{store_url}' {PKGS['A']} {PKGS['B']} {PKGS['C']} 2>&1" ) assert_count( @@ -330,12 +332,12 @@ in "Client credential provider caching failed" ) - client.succeed(f"nix path-info {PKG_A}") + client.succeed(f"nix path-info {PKGS['A']}") print(" ✓ nix copy works") print(" ✓ Credentials cached on client") - @with_test_bucket(populate_with=[PKG_A]) + @with_test_bucket(populate_with=[PKGS['A']]) def test_url_format_variations(bucket): """Test different S3 URL parameter combinations""" print("\n=== Testing URL Format Variations ===") @@ -350,7 +352,7 @@ in client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2") print(" ✓ Parameter order: endpoint before region works") - @with_test_bucket(populate_with=[PKG_A]) + @with_test_bucket(populate_with=[PKGS['A']]) def test_concurrent_fetches(bucket): """Validate thread safety with concurrent S3 operations""" print("\n=== Testing Concurrent Fetches ===") @@ -418,16 +420,16 @@ in print("\n=== Testing Compression: narinfo (gzip) ===") store_url = make_s3_url(bucket, **{'narinfo-compression': 'gzip'}) - server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_B}") + server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['B']}") - pkg_hash = get_package_hash(PKG_B) + pkg_hash = get_package_hash(PKGS['B']) verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "gzip") print(" ✓ .narinfo has Content-Encoding: gzip") # Verify client can download and decompress - client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_B}") - client.succeed(f"nix path-info {PKG_B}") + client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['B']}") + client.succeed(f"nix path-info {PKGS['B']}") print(" ✓ Client decompressed .narinfo successfully") @@ -441,9 +443,9 @@ in **{'narinfo-compression': 'xz', 'write-nar-listing': 'true', 'ls-compression': 'gzip'} ) - server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_C}") + server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['C']}") - pkg_hash = get_package_hash(PKG_C) + pkg_hash = get_package_hash(PKGS['C']) # Verify .narinfo has xz compression verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz") @@ -454,8 +456,8 @@ in print(" ✓ .ls has Content-Encoding: gzip") # Verify client can download with mixed compression - client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_C}") - client.succeed(f"nix path-info {PKG_C}") + client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['C']}") + client.succeed(f"nix path-info {PKGS['C']}") print(" ✓ Client downloaded package with mixed compression") @@ -465,9 +467,9 @@ in print("\n=== Testing Compression: disabled (default) ===") store_url = make_s3_url(bucket) - server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_A}") + server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']}") - pkg_hash = get_package_hash(PKG_A) + pkg_hash = get_package_hash(PKGS['A']) verify_no_compression(server, bucket, f"{pkg_hash}.narinfo") print(" ✓ No compression applied by default") From c1a15d1a26479380069c6ba14be9a23573e32c0f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:24:29 +0000 Subject: [PATCH 032/201] refactor(tests/nixos/s3-binary-cache-store): rename with_test_bucket to setup_s3 --- tests/nixos/s3-binary-cache-store.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 2d5c6c1c1..5ed543d89 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -135,7 +135,7 @@ in print(output) raise Exception(f"{error_msg}: expected {expected}, got {actual}") - def with_test_bucket(populate_with=[]): + def setup_s3(populate_with=[]): """ Decorator that creates/destroys a unique bucket for each test. Optionally pre-populates bucket with specified packages. @@ -162,7 +162,7 @@ in # Test Functions # ============================================================================ - @with_test_bucket() + @setup_s3() def test_credential_caching(bucket): """Verify credential providers are cached and reused""" print("\n=== Testing Credential Caching ===") @@ -182,7 +182,7 @@ in print("✓ Credential provider created once and cached") - @with_test_bucket(populate_with=[PKGS['A']]) + @setup_s3(populate_with=[PKGS['A']]) def test_fetchurl_basic(bucket): """Test builtins.fetchurl works with s3:// URLs""" print("\n=== Testing builtins.fetchurl ===") @@ -198,7 +198,7 @@ in print("✓ builtins.fetchurl works with s3:// URLs") - @with_test_bucket() + @setup_s3() def test_error_message_formatting(bucket): """Verify error messages display URLs correctly""" print("\n=== Testing Error Message Formatting ===") @@ -218,7 +218,7 @@ in print("✓ Error messages format URLs correctly") - @with_test_bucket(populate_with=[PKGS['A']]) + @setup_s3(populate_with=[PKGS['A']]) def test_fork_credential_preresolution(bucket): """Test credential pre-resolution in forked processes""" print("\n=== Testing Fork Credential Pre-resolution ===") @@ -298,7 +298,7 @@ in print(" ✓ Child uses pre-resolved credentials (no new providers)") - @with_test_bucket(populate_with=[PKGS['A'], PKGS['B'], PKGS['C']]) + @setup_s3(populate_with=[PKGS['A'], PKGS['B'], PKGS['C']]) def test_store_operations(bucket): """Test nix store info and copy operations""" print("\n=== Testing Store Operations ===") @@ -337,7 +337,7 @@ in print(" ✓ nix copy works") print(" ✓ Credentials cached on client") - @with_test_bucket(populate_with=[PKGS['A']]) + @setup_s3(populate_with=[PKGS['A']]) def test_url_format_variations(bucket): """Test different S3 URL parameter combinations""" print("\n=== Testing URL Format Variations ===") @@ -352,7 +352,7 @@ in client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2") print(" ✓ Parameter order: endpoint before region works") - @with_test_bucket(populate_with=[PKGS['A']]) + @setup_s3(populate_with=[PKGS['A']]) def test_concurrent_fetches(bucket): """Validate thread safety with concurrent S3 operations""" print("\n=== Testing Concurrent Fetches ===") @@ -414,7 +414,7 @@ in f"Expected 5 FileTransfer instances for 5 concurrent fetches, got {transfers_created}" ) - @with_test_bucket() + @setup_s3() def test_compression_narinfo_gzip(bucket): """Test narinfo compression with gzip""" print("\n=== Testing Compression: narinfo (gzip) ===") @@ -433,7 +433,7 @@ in print(" ✓ Client decompressed .narinfo successfully") - @with_test_bucket() + @setup_s3() def test_compression_mixed(bucket): """Test mixed compression (narinfo=xz, ls=gzip)""" print("\n=== Testing Compression: mixed (narinfo=xz, ls=gzip) ===") @@ -461,7 +461,7 @@ in print(" ✓ Client downloaded package with mixed compression") - @with_test_bucket() + @setup_s3() def test_compression_disabled(bucket): """Verify no compression by default""" print("\n=== Testing Compression: disabled (default) ===") From 9058d90ab2b81f77bbb844f9c783bb5a4c238d1d Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:27:03 +0000 Subject: [PATCH 033/201] refactor(tests/nixos/s3-binary-cache-store): rename populate_with to populate_bucket --- tests/nixos/s3-binary-cache-store.nix | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 5ed543d89..e7fcadb45 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -135,22 +135,22 @@ in print(output) raise Exception(f"{error_msg}: expected {expected}, got {actual}") - def setup_s3(populate_with=[]): + def setup_s3(populate_bucket=[]): """ Decorator that creates/destroys a unique bucket for each test. Optionally pre-populates bucket with specified packages. Args: - populate_with: List of packages to upload before test runs + populate_bucket: List of packages to upload before test runs """ def decorator(test_func): def wrapper(): bucket = str(uuid.uuid4()) server.succeed(f"mc mb minio/{bucket}") try: - if populate_with: + if populate_bucket: store_url = make_s3_url(bucket) - for pkg in populate_with: + for pkg in populate_bucket: server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {pkg}") test_func(bucket) finally: @@ -182,7 +182,7 @@ in print("✓ Credential provider created once and cached") - @setup_s3(populate_with=[PKGS['A']]) + @setup_s3(populate_bucket=[PKGS['A']]) def test_fetchurl_basic(bucket): """Test builtins.fetchurl works with s3:// URLs""" print("\n=== Testing builtins.fetchurl ===") @@ -218,7 +218,7 @@ in print("✓ Error messages format URLs correctly") - @setup_s3(populate_with=[PKGS['A']]) + @setup_s3(populate_bucket=[PKGS['A']]) def test_fork_credential_preresolution(bucket): """Test credential pre-resolution in forked processes""" print("\n=== Testing Fork Credential Pre-resolution ===") @@ -298,7 +298,7 @@ in print(" ✓ Child uses pre-resolved credentials (no new providers)") - @setup_s3(populate_with=[PKGS['A'], PKGS['B'], PKGS['C']]) + @setup_s3(populate_bucket=[PKGS['A'], PKGS['B'], PKGS['C']]) def test_store_operations(bucket): """Test nix store info and copy operations""" print("\n=== Testing Store Operations ===") @@ -337,7 +337,7 @@ in print(" ✓ nix copy works") print(" ✓ Credentials cached on client") - @setup_s3(populate_with=[PKGS['A']]) + @setup_s3(populate_bucket=[PKGS['A']]) def test_url_format_variations(bucket): """Test different S3 URL parameter combinations""" print("\n=== Testing URL Format Variations ===") @@ -352,7 +352,7 @@ in client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2") print(" ✓ Parameter order: endpoint before region works") - @setup_s3(populate_with=[PKGS['A']]) + @setup_s3(populate_bucket=[PKGS['A']]) def test_concurrent_fetches(bucket): """Validate thread safety with concurrent S3 operations""" print("\n=== Testing Concurrent Fetches ===") From f88c3055f8b3786f0f95ffe26813d4f3cf093247 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:36:42 +0000 Subject: [PATCH 034/201] refactor(tests/nixos/s3-binary-cache-store): clean client store in setup_s3 Add cleanup of client store in the finally block of setup_s3 decorator. Uses `nix store delete --ignore-liveness` to properly handle GC roots and only attempts deletion if the path exists. --- tests/nixos/s3-binary-cache-store.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index e7fcadb45..68d123b51 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -139,6 +139,7 @@ in """ Decorator that creates/destroys a unique bucket for each test. Optionally pre-populates bucket with specified packages. + Cleans up client store after test completion. Args: populate_bucket: List of packages to upload before test runs @@ -155,6 +156,9 @@ in test_func(bucket) finally: server.succeed(f"mc rb --force minio/{bucket}") + # Clean up client store - only delete if path exists + for pkg in PKGS.values(): + client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}") return wrapper return decorator From 4f19e63a8fa3f8079adce11a87374fa3fb8d2709 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:44:10 +0000 Subject: [PATCH 035/201] refactor(tests/nixos/s3-binary-cache-store): add --no-link to nix build commands Prevent creation of result symlinks in all nix build commands by adding the --no-link flag. --- tests/nixos/s3-binary-cache-store.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 68d123b51..d47273196 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -252,7 +252,7 @@ in """.format(id=test_id, url=test_url, hash=cache_info_hash) output = client.succeed( - f"{ENV_WITH_CREDS} nix build --debug --impure --expr '{fetchurl_expr}' 2>&1" + f"{ENV_WITH_CREDS} nix build --debug --impure --no-link --expr '{fetchurl_expr}' 2>&1" ) # Verify fork behavior @@ -392,12 +392,12 @@ in try: output = client.succeed( - f"{ENV_WITH_CREDS} nix build --debug --impure " + f"{ENV_WITH_CREDS} nix build --debug --impure --no-link " f"--expr '{concurrent_expr}' --max-jobs 5 2>&1" ) except: output = client.fail( - f"{ENV_WITH_CREDS} nix build --debug --impure " + f"{ENV_WITH_CREDS} nix build --debug --impure --no-link " f"--expr '{concurrent_expr}' --max-jobs 5 2>&1" ) From 4ae6c65bc589807c787be4b31809c15649a468f4 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:48:54 +0000 Subject: [PATCH 036/201] test(tests/nixos/s3-binary-cache-store): verify credential caching in concurrent fetches Add assertion to test_concurrent_fetches to verify that only one credential provider is created even with 5 concurrent fetches. --- tests/nixos/s3-binary-cache-store.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index d47273196..9a0975eed 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -418,6 +418,13 @@ in f"Expected 5 FileTransfer instances for 5 concurrent fetches, got {transfers_created}" ) + if providers_created != 1: + print("Debug output:") + print(output) + raise Exception( + f"Expected 1 credential provider for concurrent fetches, got {providers_created}" + ) + @setup_s3() def test_compression_narinfo_gzip(bucket): """Test narinfo compression with gzip""" From 5b4bd5bcb854f44b53119be58269f50430c55137 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:51:39 +0000 Subject: [PATCH 037/201] refactor(tests/nixos/s3-binary-cache-store): inline make_http_url fn Remove make_http_url helper function and inline its single usage. --- tests/nixos/s3-binary-cache-store.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 9a0975eed..4f5632724 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -103,10 +103,6 @@ in bucket_and_path = f"{bucket}{path}" if path else bucket return f"s3://{bucket_and_path}?{query}" - def make_http_url(path): - """Build HTTP URL for direct S3 access""" - return f"{ENDPOINT}/{path}" - def get_package_hash(pkg_path): """Extract store hash from package path""" return pkg_path.split("/")[-1].split("-")[0] @@ -208,7 +204,7 @@ in print("\n=== Testing Error Message Formatting ===") nonexistent_url = make_s3_url(bucket, path="/foo-that-does-not-exist") - expected_http_url = make_http_url(f"{bucket}/foo-that-does-not-exist") + expected_http_url = f"{ENDPOINT}/{bucket}/foo-that-does-not-exist" error_msg = client.fail( f"{ENV_WITH_CREDS} nix eval --impure --expr " From 7d0c06f921a37c85dd98dfdcd077e5aad2e9ab3e Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 23:57:51 +0000 Subject: [PATCH 038/201] feat(tests/nixos/s3-binary-cache-store): add public parameter to setup_s3 Add optional 'public' parameter to setup_s3 decorator. When set to True, the bucket will be made publicly accessible using mc anonymous set. --- tests/nixos/s3-binary-cache-store.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 4f5632724..96ca37f19 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -131,7 +131,7 @@ in print(output) raise Exception(f"{error_msg}: expected {expected}, got {actual}") - def setup_s3(populate_bucket=[]): + def setup_s3(populate_bucket=[], public=False): """ Decorator that creates/destroys a unique bucket for each test. Optionally pre-populates bucket with specified packages. @@ -139,11 +139,14 @@ in Args: populate_bucket: List of packages to upload before test runs + public: If True, make the bucket publicly accessible """ def decorator(test_func): def wrapper(): bucket = str(uuid.uuid4()) server.succeed(f"mc mb minio/{bucket}") + if public: + server.succeed(f"mc anonymous set download minio/{bucket}") try: if populate_bucket: store_url = make_s3_url(bucket) From 55ea3d3476101ef1dce6d6e88770b0b0fb12c7c3 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sun, 19 Oct 2025 00:04:30 +0000 Subject: [PATCH 039/201] test(tests/nixos/s3-binary-cache-store): test public bucket operations Add `test_public_bucket_operations` to validate that store operations work correctly on public S3 buckets without requiring credentials. Tests nix store info and nix copy operations. --- tests/nixos/s3-binary-cache-store.nix | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 96ca37f19..40804f599 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -340,6 +340,42 @@ in print(" ✓ nix copy works") print(" ✓ Credentials cached on client") + @setup_s3(populate_bucket=[PKGS['A'], PKGS['B']], public=True) + def test_public_bucket_operations(bucket): + """Test store operations on public bucket without credentials""" + print("\n=== Testing Public Bucket Operations ===") + + store_url = make_s3_url(bucket) + + # Verify store info works without credentials + client.succeed(f"nix store info --store '{store_url}' >&2") + print(" ✓ nix store info works without credentials") + + # Get and validate store info JSON + info_json = client.succeed(f"nix store info --json --store '{store_url}'") + store_info = json.loads(info_json) + + if not store_info.get("url"): + raise Exception("Store should have a URL") + + print(f" ✓ Store URL: {store_info['url']}") + + # Verify packages are not yet in client store + client.fail(f"nix path-info {PKGS['A']}") + client.fail(f"nix path-info {PKGS['B']}") + + # Test copy from public bucket without credentials + client.succeed( + f"nix copy --debug --no-check-sigs " + f"--from '{store_url}' {PKGS['A']} {PKGS['B']} 2>&1" + ) + + # Verify packages were copied successfully + client.succeed(f"nix path-info {PKGS['A']}") + client.succeed(f"nix path-info {PKGS['B']}") + + print(" ✓ nix copy from public bucket works without credentials") + @setup_s3(populate_bucket=[PKGS['A']]) def test_url_format_variations(bucket): """Test different S3 URL parameter combinations""" @@ -506,6 +542,7 @@ in test_error_message_formatting() test_fork_credential_preresolution() test_store_operations() + test_public_bucket_operations() test_url_format_variations() test_concurrent_fetches() test_compression_narinfo_gzip() From d9c808f8a76ef35050d3aa65e1973ab7d69ed48e Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sun, 19 Oct 2025 00:21:46 +0000 Subject: [PATCH 040/201] refactor(tests/nixos/s3-binary-cache-store): add verify_packages_in_store helper --- tests/nixos/s3-binary-cache-store.nix | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 40804f599..b1995bd3a 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -131,6 +131,22 @@ in print(output) raise Exception(f"{error_msg}: expected {expected}, got {actual}") + def verify_packages_in_store(machine, pkg_paths, should_exist=True): + """ + Verify whether packages exist in the store. + + Args: + machine: The machine to check on + pkg_paths: List of package paths to check (or single path) + should_exist: If True, verify packages exist; if False, verify they don't + """ + paths = [pkg_paths] if isinstance(pkg_paths, str) else pkg_paths + for pkg in paths: + if should_exist: + machine.succeed(f"nix path-info {pkg}") + else: + machine.fail(f"nix path-info {pkg}") + def setup_s3(populate_bucket=[], public=False): """ Decorator that creates/destroys a unique bucket for each test. @@ -321,7 +337,7 @@ in print(f" ✓ Store URL: {store_info['url']}") # Test copy from store - client.fail(f"nix path-info {PKGS['A']}") + verify_packages_in_store(client, PKGS['A'], should_exist=False) output = client.succeed( f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs " @@ -335,7 +351,7 @@ in "Client credential provider caching failed" ) - client.succeed(f"nix path-info {PKGS['A']}") + verify_packages_in_store(client, [PKGS['A'], PKGS['B'], PKGS['C']]) print(" ✓ nix copy works") print(" ✓ Credentials cached on client") @@ -361,8 +377,7 @@ in print(f" ✓ Store URL: {store_info['url']}") # Verify packages are not yet in client store - client.fail(f"nix path-info {PKGS['A']}") - client.fail(f"nix path-info {PKGS['B']}") + verify_packages_in_store(client, [PKGS['A'], PKGS['B']], should_exist=False) # Test copy from public bucket without credentials client.succeed( @@ -371,8 +386,7 @@ in ) # Verify packages were copied successfully - client.succeed(f"nix path-info {PKGS['A']}") - client.succeed(f"nix path-info {PKGS['B']}") + verify_packages_in_store(client, [PKGS['A'], PKGS['B']]) print(" ✓ nix copy from public bucket works without credentials") @@ -475,7 +489,7 @@ in # Verify client can download and decompress client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['B']}") - client.succeed(f"nix path-info {PKGS['B']}") + verify_packages_in_store(client, PKGS['B']) print(" ✓ Client decompressed .narinfo successfully") @@ -503,7 +517,7 @@ in # Verify client can download with mixed compression client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['C']}") - client.succeed(f"nix path-info {PKGS['C']}") + verify_packages_in_store(client, PKGS['C']) print(" ✓ Client downloaded package with mixed compression") From e33cd5aa38c5760b5c0cda08aad2b5e9eb7768ff Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Oct 2025 14:08:34 +0200 Subject: [PATCH 041/201] Clarify unlocked input warning message The previous message was vague about what "deprecated" meant and why unlocked inputs with NAR hashes "may not be reproducible". It also used "verifiable" which was confusing. The new message makes it clear that the NAR hash provides verification (is checked by NAR hash) and explicitly states the failure modes: garbage collection and sharing. --- src/libexpr/primops/fetchTree.cc | 4 ++-- src/libflake/lockfile.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ad76af5b5..b49bd02e7 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -199,8 +199,8 @@ static void fetchTree( if (state.settings.pureEval && !input.isLocked()) { if (input.getNarHash()) warn( - "Input '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. " - "This is deprecated since such inputs are verifiable but may not be reproducible.", + "Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " + "This is not reproducible and will break after garbage collection or when shared.", input.to_string()); else state diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index d3dac19c5..d0d339f9f 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -77,8 +77,8 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann: if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) { if (lockedRef.input.getNarHash()) warn( - "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. " - "This is deprecated since such inputs are verifiable but may not be reproducible.", + "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " + "This is not reproducible and will break after garbage collection or when shared.", lockedRef.to_string()); else throw Error( From c663f7ec79aa21037dabc0df130eb6f3e98d10c4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Oct 2025 21:03:13 +0300 Subject: [PATCH 042/201] libstore: Fix reentrancy in AwsCredentialProviderImpl::getCredentialsRaw Old code would do very much incorrect reentrancy crimes (trying to do an erase inside the emplace callback). This would fail miserably with an assertion in Boost: terminating due to unexpected unrecoverable internal error: Assertion '(!find(px))&&("reentrancy not allowed")' failed in boost::unordered::detail::foa::entry_trace::entry_trace(const void *) at include/boost/unordered/detail/foa/reentrancy_check.hpp:33 This is trivially reproduced by using any S3 URL with a non-empty profile: nix-prefetch-url "s3://happy/crash?profile=default" --- src/libstore/aws-creds.cc | 82 +++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index d58293560..ff7b0f0ef 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -96,67 +96,57 @@ public: } } + std::shared_ptr createProviderForProfile(const std::string & profile); + private: Aws::Crt::ApiHandle apiHandle; boost::concurrent_flat_map> credentialProviderCache; }; +std::shared_ptr +AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) +{ + debug( + "[pid=%d] creating new AWS credential provider for profile '%s'", + getpid(), + profile.empty() ? "(default)" : profile.c_str()); + + if (profile.empty()) { + Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config; + config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); + return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config); + } + + Aws::Crt::Auth::CredentialsProviderProfileConfig config; + config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); + // This is safe because the underlying C library will copy this string + // c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220 + config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str()); + return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config); +} + AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile) { - // Get or create credential provider with caching std::shared_ptr provider; - // Use try_emplace_and_cvisit for atomic get-or-create - // This prevents race conditions where multiple threads create providers credentialProviderCache.try_emplace_and_cvisit( profile, - nullptr, // Placeholder - will be replaced in f1 before any thread can see it - [&](auto & kv) { - // f1: Called atomically during insertion with non-const reference - // Other threads are blocked until we finish, so nullptr is never visible - debug( - "[pid=%d] creating new AWS credential provider for profile '%s'", - getpid(), - profile.empty() ? "(default)" : profile.c_str()); + nullptr, + [&](auto & kv) { provider = kv.second = createProviderForProfile(profile); }, + [&](const auto & kv) { provider = kv.second; }); - try { - if (profile.empty()) { - Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config; - config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); - kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config); - } else { - Aws::Crt::Auth::CredentialsProviderProfileConfig config; - config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); - // This is safe because the underlying C library will copy this string - // c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220 - config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str()); - kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config); - } - - if (!kv.second) { - throw AwsAuthError( - "Failed to create AWS credentials provider for %s", - profile.empty() ? "default profile" : fmt("profile '%s'", profile)); - } - - provider = kv.second; - } catch (Error & e) { - // Exception during creation - remove the entry to allow retry - credentialProviderCache.erase(profile); - e.addTrace({}, "for AWS profile: %s", profile.empty() ? "(default)" : profile); - throw; - } catch (...) { - // Non-Error exception - still need to clean up - credentialProviderCache.erase(profile); - throw; - } - }, - [&](const auto & kv) { - // f2: Called if key already exists (const reference) - provider = kv.second; + if (!provider) { + credentialProviderCache.erase_if(profile, [](const auto & kv) { + [[maybe_unused]] auto [_, provider] = kv; + return !provider; }); + throw AwsAuthError( + "Failed to create AWS credentials provider for %s", + profile.empty() ? "default profile" : fmt("profile '%s'", profile)); + } + return getCredentialsFromProvider(provider); } From 6c9083db2c4d68c3a3719a816da48b68541a4a72 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Oct 2025 13:40:19 +0200 Subject: [PATCH 043/201] Use a smaller buffer --- src/libutil/serialise.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index bdce956f3..47a00c8d6 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -112,7 +112,7 @@ std::string Source::drain() void Source::skip(size_t len) { - std::array buf; + std::array buf; while (len) { auto n = read(buf.data(), std::min(len, buf.size())); assert(n <= len); From a91b7875249e84cb0b3a836b5fd59267481fa0cd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Oct 2025 21:08:49 +0300 Subject: [PATCH 044/201] libutil: Add alignUp helper function --- src/libutil-tests/alignment.cc | 18 ++++++++++++++++++ src/libutil-tests/meson.build | 1 + src/libutil/include/nix/util/alignment.hh | 23 +++++++++++++++++++++++ src/libutil/include/nix/util/meson.build | 1 + 4 files changed, 43 insertions(+) create mode 100644 src/libutil-tests/alignment.cc create mode 100644 src/libutil/include/nix/util/alignment.hh diff --git a/src/libutil-tests/alignment.cc b/src/libutil-tests/alignment.cc new file mode 100644 index 000000000..bef0c435d --- /dev/null +++ b/src/libutil-tests/alignment.cc @@ -0,0 +1,18 @@ +#include "nix/util/alignment.hh" + +#include + +namespace nix { + +TEST(alignUp, value) +{ + for (uint64_t i = 1; i <= 8; ++i) + EXPECT_EQ(alignUp(i, 8), 8); +} + +TEST(alignUp, notAPowerOf2) +{ + ASSERT_DEATH({ alignUp(1u, 42); }, "alignment must be a power of 2"); +} + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index d84dbbb68..c75f4d90a 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -44,6 +44,7 @@ config_priv_h = configure_file( subdir('nix-meson-build-support/common') sources = files( + 'alignment.cc', 'archive.cc', 'args.cc', 'base-n.cc', diff --git a/src/libutil/include/nix/util/alignment.hh b/src/libutil/include/nix/util/alignment.hh new file mode 100644 index 000000000..a4e5af4d6 --- /dev/null +++ b/src/libutil/include/nix/util/alignment.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include +#include +#include +#include + +namespace nix { + +/// Aligns val upwards to be a multiple of alignment. +/// +/// @pre alignment must be a power of 2. +template + requires std::is_unsigned_v +constexpr T alignUp(T val, unsigned alignment) +{ + assert(std::has_single_bit(alignment) && "alignment must be a power of 2"); + T mask = ~(T{alignment} - 1u); + return (val + alignment - 1) & mask; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index dcfaa8e3f..9a606e15d 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -4,6 +4,7 @@ include_dirs = [ include_directories('../..') ] headers = files( 'abstract-setting-to-json.hh', + 'alignment.hh', 'ansicolor.hh', 'archive.hh', 'args.hh', From 22c73868c396ceb189ff2638768b3eea30120ded Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Oct 2025 21:15:11 +0300 Subject: [PATCH 045/201] libutil/archive: Use alignUp With this change it's much more apparent what's going on. --- src/libutil/archive.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index b8fef9ef3..73ec0cab7 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,6 +6,7 @@ #include // for strcasecmp #include "nix/util/archive.hh" +#include "nix/util/alignment.hh" #include "nix/util/config-global.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/util/source-path.hh" @@ -133,7 +134,7 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) sink.preallocateContents(size); if (sink.skipContents) { - source.skip(size + (size % 8 ? 8 - (size % 8) : 0)); + source.skip(alignUp(size, 8)); return; } From e3b3f05e5d0ee92ccee4e8679af45609e786d149 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 20:45:43 +0000 Subject: [PATCH 046/201] fix(nix-prefetch-url): correctly extract filename from URLs with query parameters Previously, `prefetchFile()` used `baseNameOf()` directly on the URL string to extract the filename. This caused issues with URLs containing query parameters that include slashes, such as S3 URLs with custom endpoints: ``` s3://bucket/file.txt?endpoint=http://server:9000 ``` The `baseNameOf()` function naively searches for the rightmost `/` in the entire string, which would find the `/` in `http://server:9000` and extract `server:9000®ion=...` as the filename. This resulted in invalid store path names containing illegal characters like `:`. This commit fixes the issue by: 1. Adding a `VerbatimURL::lastPathSegment()` method that extracts the last non-empty path segment from a URL, using `pathSegments(true)` to filter empty segments 2. Changing `prefetchFile()` to accept `const VerbatimURL &` and use the new `lastPathSegment()` method instead of manual path parsing 3. Adding early validation with `checkName()` to fail quickly on invalid filenames 4. Maintains backward compatibility by falling back to `baseNameOf()` for unparsable `VerbatimURL`s --- src/libutil/include/nix/util/url.hh | 11 +++++++++++ src/libutil/url.cc | 18 ++++++++++++++++++ src/nix/prefetch.cc | 22 +++++++++++++++------- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/libutil/include/nix/util/url.hh b/src/libutil/include/nix/util/url.hh index 4ed80feb3..1fc8c3f2b 100644 --- a/src/libutil/include/nix/util/url.hh +++ b/src/libutil/include/nix/util/url.hh @@ -408,6 +408,17 @@ struct VerbatimURL [](const ParsedURL & url) -> std::string_view { return url.scheme; }}, raw); } + + /** + * Get the last non-empty path segment from the URL. + * + * This is useful for extracting filenames from URLs. + * For example, "https://example.com/path/to/file.txt?query=value" + * returns "file.txt". + * + * @return The last non-empty path segment, or std::nullopt if no such segment exists. + */ + std::optional lastPathSegment() const; }; std::ostream & operator<<(std::ostream & os, const VerbatimURL & url); diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 7410e4062..538792463 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -4,6 +4,7 @@ #include "nix/util/split.hh" #include "nix/util/canon-path.hh" #include "nix/util/strings-inline.hh" +#include "nix/util/file-system.hh" #include @@ -440,4 +441,21 @@ std::ostream & operator<<(std::ostream & os, const VerbatimURL & url) return os; } +std::optional VerbatimURL::lastPathSegment() const +{ + try { + auto parsedUrl = parsed(); + auto segments = parsedUrl.pathSegments(/*skipEmpty=*/true); + if (std::ranges::empty(segments)) + return std::nullopt; + return segments.back(); + } catch (BadURL &) { + // Fall back to baseNameOf for unparsable URLs + auto name = baseNameOf(to_string()); + if (name.empty()) + return std::nullopt; + return std::string{name}; + } +} + } // namespace nix diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 18abfa0aa..d875f8e4b 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -13,6 +13,8 @@ #include "nix/cmd/misc-store-flags.hh" #include "nix/util/terminal.hh" #include "nix/util/environment-variables.hh" +#include "nix/util/url.hh" +#include "nix/store/path.hh" #include "man-pages.hh" @@ -56,7 +58,7 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) std::tuple prefetchFile( ref store, - std::string_view url, + const VerbatimURL & url, std::optional name, HashAlgorithm hashAlgo, std::optional expectedHash, @@ -68,9 +70,15 @@ std::tuple prefetchFile( /* Figure out a name in the Nix store. */ if (!name) { - name = baseNameOf(url); - if (name->empty()) - throw Error("cannot figure out file name for '%s'", url); + name = url.lastPathSegment(); + if (!name || name->empty()) + throw Error("cannot figure out file name for '%s'", url.to_string()); + } + try { + checkName(*name); + } catch (BadStorePathName & e) { + e.addTrace({}, "file name '%s' was extracted from URL '%s'", *name, url.to_string()); + throw; } std::optional storePath; @@ -105,14 +113,14 @@ std::tuple prefetchFile( FdSink sink(fd.get()); - FileTransferRequest req(VerbatimURL{url}); + FileTransferRequest req(url); req.decompress = false; getFileTransfer()->download(std::move(req), sink); } /* Optionally unpack the file. */ if (unpack) { - Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url)); + Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url.to_string())); auto unpacked = (tmpDir.path() / "unpacked").string(); createDirs(unpacked); unpackTarfile(tmpFile.string(), unpacked); @@ -128,7 +136,7 @@ std::tuple prefetchFile( } } - Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); + Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url.to_string())); auto info = store->addToStoreSlow( *name, PosixSourceAccessor::createAtRoot(tmpFile), method, hashAlgo, {}, expectedHash); From 0f28c76a4471b20e2257cff408ae0f25a055d283 Mon Sep 17 00:00:00 2001 From: David McFarland Date: Mon, 20 Oct 2025 15:40:05 -0300 Subject: [PATCH 047/201] nix/develop: Strip outputChecks when structuredAttrs is enabled --- src/nix/develop.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 28d0a7080..d23dce10b 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -254,10 +254,15 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore drv.args = {store->printStorePath(getEnvShPath)}; /* Remove derivation checks. */ - drv.env.erase("allowedReferences"); - drv.env.erase("allowedRequisites"); - drv.env.erase("disallowedReferences"); - drv.env.erase("disallowedRequisites"); + if (drv.structuredAttrs) { + drv.structuredAttrs->structuredAttrs.erase("outputChecks"); + } else { + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + } + drv.env.erase("name"); /* Rehash and write the derivation. FIXME: would be nice to use From 1b1d7e30470e2979674214b021e8d13fd0e7df93 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 18 Oct 2025 20:48:50 +0000 Subject: [PATCH 048/201] test(nixos): add nix-prefetch-url test for S3 URLs with query parameters Adds a comprehensive test to verify that `nix-prefetch-url` correctly handles S3 URLs with query parameters (e.g., custom endpoints and regions). Previously, nix-prefetch-url would fail with "invalid store path" errors when given S3 URLs with query parameters like `?endpoint=http://server:9000®ion=eu-west-1`, because it incorrectly extracted the filename from the query parameters instead of the path. --- tests/nixos/s3-binary-cache-store.nix | 64 +++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index b1995bd3a..981fab868 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -534,6 +534,69 @@ in print(" ✓ No compression applied by default") + @setup_s3() + def test_nix_prefetch_url(bucket): + """Test that nix-prefetch-url retrieves actual file content from S3, not empty files (issue #8862)""" + print("\n=== Testing nix-prefetch-url S3 Content Retrieval (issue #8862) ===") + + # Create a test file with known content + test_content = "This is test content to verify S3 downloads work correctly!\n" + test_file_size = len(test_content) + + server.succeed(f"echo -n '{test_content}' > /tmp/test-file.txt") + + # Upload to S3 + server.succeed(f"mc cp /tmp/test-file.txt minio/{bucket}/test-file.txt") + + # Calculate expected hash + expected_hash = server.succeed( + "nix hash file --type sha256 --base32 /tmp/test-file.txt" + ).strip() + + print(f" ✓ Uploaded test file to S3 ({test_file_size} bytes)") + + # Use nix-prefetch-url to download from S3 + s3_url = make_s3_url(bucket, path="/test-file.txt") + + prefetch_output = client.succeed( + f"{ENV_WITH_CREDS} nix-prefetch-url --print-path '{s3_url}'" + ) + + # Extract hash and store path + # With --print-path, output is: \n + lines = prefetch_output.strip().split('\n') + prefetch_hash = lines[0] # First line is the hash + store_path = lines[1] # Second line is the store path + + # Verify hash matches + if prefetch_hash != expected_hash: + raise Exception( + f"Hash mismatch: expected {expected_hash}, got {prefetch_hash}" + ) + + print(" ✓ nix-prefetch-url completed with correct hash") + + # Verify the downloaded file is NOT empty (the bug in #8862) + file_size = int(client.succeed(f"stat -c %s {store_path}").strip()) + + if file_size == 0: + raise Exception("Downloaded file is EMPTY - issue #8862 regression detected!") + + if file_size != test_file_size: + raise Exception( + f"File size mismatch: expected {test_file_size}, got {file_size}" + ) + + print(f" ✓ File has correct size ({file_size} bytes, not empty)") + + # Verify actual content matches by comparing hashes instead of printing entire file + downloaded_hash = client.succeed(f"nix hash file --type sha256 --base32 {store_path}").strip() + + if downloaded_hash != expected_hash: + raise Exception(f"Content hash mismatch: expected {expected_hash}, got {downloaded_hash}") + + print(" ✓ File content verified correct (hash matches)") + # ============================================================================ # Main Test Execution # ============================================================================ @@ -562,6 +625,7 @@ in test_compression_narinfo_gzip() test_compression_mixed() test_compression_disabled() + test_nix_prefetch_url() print("\n" + "="*80) print("✓ All S3 Binary Cache Store Tests Passed!") From 5e7ee808de8bdc353f80401e8fd8a310a2622f4b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Sep 2025 08:32:26 -0400 Subject: [PATCH 049/201] `nlohmann::json` instance and JSON Schema for `Hash` Improving and codifying our experimental JSON interfacing. Co-Authored-By: Robert Hensing --- doc/manual/meson.build | 2 +- doc/manual/package.nix | 2 + doc/manual/source/protocols/json/hash.md | 26 ++++ .../source/protocols/json/schema/hash-v1 | 1 + .../source/protocols/json/schema/hash-v1.yaml | 30 ++++- src/json-schema-checks/hash | 1 + src/json-schema-checks/meson.build | 10 ++ src/json-schema-checks/package.nix | 1 + .../data/hash/blake3-base64.json | 5 + .../data/hash/sha256-base16.json | 5 + .../data/hash/sha256-base64.json | 5 + src/libutil-tests/data/hash/sha256-nix32.json | 5 + src/libutil-tests/data/hash/simple.json | 5 + src/libutil-tests/hash.cc | 111 ++++++++++++++++-- src/libutil/hash.cc | 48 ++++++-- src/libutil/include/nix/util/hash.hh | 19 ++- 16 files changed, 252 insertions(+), 24 deletions(-) create mode 120000 doc/manual/source/protocols/json/schema/hash-v1 create mode 120000 src/json-schema-checks/hash create mode 100644 src/libutil-tests/data/hash/blake3-base64.json create mode 100644 src/libutil-tests/data/hash/sha256-base16.json create mode 100644 src/libutil-tests/data/hash/sha256-base64.json create mode 100644 src/libutil-tests/data/hash/sha256-nix32.json create mode 100644 src/libutil-tests/data/hash/simple.json diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 7090c949c..fdea40098 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -88,7 +88,7 @@ manual = custom_target( @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml - @4@ -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ + @4@ -r -L --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 rm -rf @2@/manual mv @2@/html @2@/manual diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 7b94721ae..30486869e 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -33,6 +33,8 @@ mkMesonDerivation (finalAttrs: { fileset.difference (fileset.unions [ ../../.version + # For example JSON + ../../src/libutil-tests/data/hash # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/protocols/json/hash.md b/doc/manual/source/protocols/json/hash.md index d2bdf1062..efd920086 100644 --- a/doc/manual/source/protocols/json/hash.md +++ b/doc/manual/source/protocols/json/hash.md @@ -1,5 +1,31 @@ {{#include hash-v1-fixed.md}} +## Examples + +### SHA-256 with Base64 encoding + +```json +{{#include schema/hash-v1/sha256-base64.json}} +``` + +### SHA-256 with Base16 (hexadecimal) encoding + +```json +{{#include schema/hash-v1/sha256-base16.json}} +``` + +### SHA-256 with Nix32 encoding + +```json +{{#include schema/hash-v1/sha256-nix32.json}} +``` + +### BLAKE3 with Base64 encoding + +```json +{{#include schema/hash-v1/blake3-base64.json}} +``` + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 126e666e9..27895d42a 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -12,3 +12,6 @@ s/\\`/`/g # As we have more such relative links, more replacements of this nature # should appear below. s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g +s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g +s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g +s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 191ec6dbe..f79667961 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -10,6 +10,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ 'hash-v1', + 'content-address-v1', 'derivation-v3', 'deriving-path-v1', ] diff --git a/doc/manual/source/protocols/json/schema/content-address-v1 b/doc/manual/source/protocols/json/schema/content-address-v1 new file mode 120000 index 000000000..35a0dd865 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/content-address-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/content-address \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/content-address-v1.yaml b/doc/manual/source/protocols/json/schema/content-address-v1.yaml new file mode 100644 index 000000000..d0f759201 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/content-address-v1.yaml @@ -0,0 +1,55 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/content-address-v1.json" +title: Content Address +description: | + This schema describes the JSON representation of Nix's `ContentAddress` type, which conveys information about [content-addressing store objects](@docroot@/store/store-object/content-address.md). + + > **Note** + > + > For current methods of content addressing, this data type is a bit suspicious, because it is neither simply a content address of a file system object (the `method` is richer), nor simply a content address of a store object (the `hash` doesn't account for the references). + > It should thus only be used in contexts where the references are also known / otherwise made tamper-resistant. + + + +type: object +properties: + method: + "$ref": "#/$defs/method" + hash: + title: Content Address + description: | + This would be the content-address itself. + + For all current methods, this is just a content address of the file system object of the store object, [as described in the store chapter](@docroot@/store/file-system-object/content-address.md), and not of the store object as a whole. + In particular, the references of the store object are *not* taken into account with this hash (and currently-supported methods). + "$ref": "./hash-v1.yaml" +required: +- method +- hash +additionalProperties: false +"$defs": + method: + type: string + enum: [flat, nar, text, git] + title: Content-Addressing Method + description: | + A string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) (provided the contents are a single file) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index 7c92d475d..c950b839f 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -1,5 +1,5 @@ -"$schema": http://json-schema.org/draft-04/schema# -"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json" title: Derivation description: | Experimental JSON representation of a Nix derivation (version 3). @@ -154,19 +154,10 @@ properties: The output path, if known in advance. method: - type: string - title: Content addressing method - enum: [flat, nar, text, git] + "$ref": "./content-address-v1.yaml#/$defs/method" description: | For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. - - Valid method strings are: - - - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) - - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) - - [`text`](@docroot@/store/store-object/content-address.md#method-text) - - [`git`](@docroot@/store/store-object/content-address.md#method-git) - + See the linked original definition for further details. hashAlgo: title: Hash algorithm "$ref": "./hash-v1.yaml#/$defs/algorithm" diff --git a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml index 9c0350d3d..7fd74941e 100644 --- a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml @@ -1,5 +1,5 @@ -"$schema": http://json-schema.org/draft-04/schema# -"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json" title: Deriving Path description: | This schema describes the JSON representation of Nix's [Deriving Path](@docroot@/store/derivation/index.md#deriving-path). diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml index 844959bcd..316fb6d73 100644 --- a/doc/manual/source/protocols/json/schema/hash-v1.yaml +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -1,5 +1,5 @@ -"$schema": http://json-schema.org/draft-04/schema# -"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json" title: Hash description: | A cryptographic hash value used throughout Nix for content addressing and integrity verification. diff --git a/src/json-schema-checks/content-address b/src/json-schema-checks/content-address new file mode 120000 index 000000000..194a265a1 --- /dev/null +++ b/src/json-schema-checks/content-address @@ -0,0 +1 @@ +../../src/libstore-tests/data/content-address \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 09da8770b..745fb5ffa 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -30,6 +30,14 @@ schemas = [ 'blake3-base64.json', ], }, + { + 'stem' : 'content-address', + 'schema' : schema_dir / 'content-address-v1.yaml', + 'files' : [ + 'text.json', + 'nar.json', + ], + }, { 'stem' : 'derivation', 'schema' : schema_dir / 'derivation-v3.yaml', @@ -73,8 +81,6 @@ foreach schema : schemas stem + '-schema-valid', jv, args : [ - '--map', - './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml', 'http://json-schema.org/draft-04/schema', schema_file, ], diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index cf4e4cb19..6a76c8b28 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -21,6 +21,7 @@ mkMesonDerivation (finalAttrs: { ../../.version ../../doc/manual/source/protocols/json/schema ../../src/libutil-tests/data/hash + ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/derivation ../../src/libstore-tests/data/derived-path ./. diff --git a/src/libstore-tests/content-address.cc b/src/libstore-tests/content-address.cc index 51d591c38..0474fb2e0 100644 --- a/src/libstore-tests/content-address.cc +++ b/src/libstore-tests/content-address.cc @@ -1,6 +1,7 @@ #include #include "nix/store/content-address.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { @@ -8,33 +9,93 @@ namespace nix { * ContentAddressMethod::parse, ContentAddressMethod::render * --------------------------------------------------------------------------*/ -TEST(ContentAddressMethod, testRoundTripPrintParse_1) +static auto methods = ::testing::Values( + std::pair{ContentAddressMethod::Raw::Text, "text"}, + std::pair{ContentAddressMethod::Raw::Flat, "flat"}, + std::pair{ContentAddressMethod::Raw::NixArchive, "nar"}, + std::pair{ContentAddressMethod::Raw::Git, "git"}); + +struct ContentAddressMethodTest : ::testing::Test, + ::testing::WithParamInterface> +{}; + +TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1) { - for (ContentAddressMethod cam : { - ContentAddressMethod::Raw::Text, - ContentAddressMethod::Raw::Flat, - ContentAddressMethod::Raw::NixArchive, - ContentAddressMethod::Raw::Git, - }) { - EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); - } + auto & [cam, _] = GetParam(); + EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); } -TEST(ContentAddressMethod, testRoundTripPrintParse_2) +TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2) { - for (const std::string_view camS : { - "text", - "flat", - "nar", - "git", - }) { - EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); - } + auto & [cam, camS] = GetParam(); + EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); } +INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods); + TEST(ContentAddressMethod, testParseContentAddressMethodOptException) { EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError); } +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +class ContentAddressTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "content-address"; + +public: + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct ContentAddressJsonTest : ContentAddressTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(ContentAddressJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(ContentAddressJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + ContentAddressJSON, + ContentAddressJsonTest, + ::testing::Values( + std::pair{ + "text", + ContentAddress{ + .method = ContentAddressMethod::Raw::Text, + .hash = hashString(HashAlgorithm::SHA256, "asdf"), + }, + }, + std::pair{ + "nar", + ContentAddress{ + .method = ContentAddressMethod::Raw::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "qwer"), + }, + })); + } // namespace nix diff --git a/src/libstore-tests/data/content-address/nar.json b/src/libstore-tests/data/content-address/nar.json new file mode 100644 index 000000000..21e065cd3 --- /dev/null +++ b/src/libstore-tests/data/content-address/nar.json @@ -0,0 +1,8 @@ +{ + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "9vLqj0XYoFfJVmoz+ZR02i5camYE1zYSFlDicwxvsKM=" + }, + "method": "nar" +} diff --git a/src/libstore-tests/data/content-address/text.json b/src/libstore-tests/data/content-address/text.json new file mode 100644 index 000000000..04bc8ac20 --- /dev/null +++ b/src/libstore-tests/data/content-address/text.json @@ -0,0 +1,8 @@ +{ + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts=" + }, + "method": "text" +} diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 9a57e3aa6..497c2c5b4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,6 +1,7 @@ #include "nix/util/args.hh" #include "nix/store/content-address.hh" #include "nix/util/split.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -300,3 +301,36 @@ Hash ContentAddressWithReferences::getHash() const } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +ContentAddressMethod adl_serializer::from_json(const json & json) +{ + return ContentAddressMethod::parse(getString(json)); +} + +void adl_serializer::to_json(json & json, const ContentAddressMethod & m) +{ + json = m.render(); +} + +ContentAddress adl_serializer::from_json(const json & json) +{ + auto obj = getObject(json); + return { + .method = adl_serializer::from_json(valueAt(obj, "method")), + .hash = valueAt(obj, "hash"), + }; +} + +void adl_serializer::to_json(json & json, const ContentAddress & ca) +{ + json = { + {"method", ca.method}, + {"hash", ca.hash}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/content-address.hh b/src/libstore/include/nix/store/content-address.hh index 0a3dc79bd..41ccc69ae 100644 --- a/src/libstore/include/nix/store/content-address.hh +++ b/src/libstore/include/nix/store/content-address.hh @@ -6,6 +6,7 @@ #include "nix/store/path.hh" #include "nix/util/file-content-address.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -308,4 +309,15 @@ struct ContentAddressWithReferences Hash getHash() const; }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::ContentAddressMethod) +JSON_IMPL(nix::ContentAddress) From 3915b3a111ffe42d1ac9c8162b5506fa7678464f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 08:10:20 +0000 Subject: [PATCH 107/201] feat(libstore/s3-binary-cache-store): implement `abortMultipartUpload()` Implement `abortMultipartUpload()` for cleaning up incomplete multipart uploads on error: - Constructs URL with `?uploadId=ID` query parameter - Issues `DELETE` request to abort the multipart upload --- src/libstore/s3-binary-cache-store.cc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 5d97fb0fd..98f742c70 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -26,6 +26,14 @@ public: private: ref s3Config; + + /** + * Abort a multipart upload + * + * @see + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html#API_AbortMultipartUpload_RequestSyntax + */ + void abortMultipartUpload(std::string_view key, std::string_view uploadId); }; void S3BinaryCacheStore::upsertFile( @@ -37,6 +45,19 @@ void S3BinaryCacheStore::upsertFile( HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint); } +void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::DELETE; + + getFileTransfer()->enqueueFileTransfer(req).get(); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"}; From 5e220271e2dbafb5205684354057aeaa4a58a5c6 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 22:38:43 +0000 Subject: [PATCH 108/201] feat(libstore): add scanForReferencesDeep for per-file reference tracking Introduces `scanForReferencesDeep` to provide per-file granularity when scanning for store path references, enabling better diagnostics for cycle detection and `nix why-depends --precise`. --- src/libstore-tests/references.cc | 143 ++++++++++++++++++ .../include/nix/store/path-references.hh | 57 +++++++ src/libstore/path-references.cc | 90 +++++++++++ 3 files changed, 290 insertions(+) diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index 27ecad08f..9cecd573e 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -1,4 +1,6 @@ #include "nix/store/references.hh" +#include "nix/store/path-references.hh" +#include "nix/util/memory-source-accessor.hh" #include @@ -79,4 +81,145 @@ TEST(references, scan) } } +TEST(references, scanForReferencesDeep) +{ + using File = MemorySourceAccessor::File; + + // Create store paths to search for + StorePath path1{"dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo"}; + StorePath path2{"zc842j0rz61mjsp3h3wp5ly71ak6qgdn-bar"}; + StorePath path3{"a5cn2i4b83gnsm60d38l3kgb8qfplm11-baz"}; + + StorePathSet refs{path1, path2, path3}; + + std::string_view hash1 = path1.hashPart(); + std::string_view hash2 = path2.hashPart(); + std::string_view hash3 = path3.hashPart(); + + // Create an in-memory file system with various reference patterns + auto accessor = make_ref(); + accessor->root = File::Directory{ + .contents{ + { + // file1.txt: contains hash1 + "file1.txt", + File::Regular{ + .contents = "This file references " + hash1 + " in its content", + }, + }, + { + // file2.txt: contains hash2 and hash3 + "file2.txt", + File::Regular{ + .contents = "Multiple refs: " + hash2 + " and also " + hash3, + }, + }, + { + // file3.txt: contains no references + "file3.txt", + File::Regular{ + .contents = "This file has no store path references at all", + }, + }, + { + // subdir: a subdirectory + "subdir", + File::Directory{ + .contents{ + { + // subdir/file4.txt: contains hash1 again + "file4.txt", + File::Regular{ + .contents = "Subdirectory file with " + hash1, + }, + }, + }, + }, + }, + { + // link1: a symlink that contains a reference in its target + "link1", + File::Symlink{ + .target = hash2 + "-target", + }, + }, + }, + }; + + // Test the callback-based API + { + std::map foundRefs; + + scanForReferencesDeep(*accessor, CanonPath::root, refs, [&](FileRefScanResult result) { + foundRefs[std::move(result.filePath)] = std::move(result.foundRefs); + }); + + // Verify we found the expected references + EXPECT_EQ(foundRefs.size(), 4); // file1, file2, file4, link1 + + // Check file1.txt found path1 + { + CanonPath f1Path("/file1.txt"); + auto it = foundRefs.find(f1Path); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 1); + EXPECT_TRUE(it->second.count(path1)); + } + + // Check file2.txt found path2 and path3 + { + CanonPath f2Path("/file2.txt"); + auto it = foundRefs.find(f2Path); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 2); + EXPECT_TRUE(it->second.count(path2)); + EXPECT_TRUE(it->second.count(path3)); + } + + // Check file3.txt is not in results (no refs) + { + CanonPath f3Path("/file3.txt"); + EXPECT_FALSE(foundRefs.count(f3Path)); + } + + // Check subdir/file4.txt found path1 + { + CanonPath f4Path("/subdir/file4.txt"); + auto it = foundRefs.find(f4Path); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 1); + EXPECT_TRUE(it->second.count(path1)); + } + + // Check symlink found path2 + { + CanonPath linkPath("/link1"); + auto it = foundRefs.find(linkPath); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 1); + EXPECT_TRUE(it->second.count(path2)); + } + } + + // Test the map-based convenience API + { + auto results = scanForReferencesDeep(*accessor, CanonPath::root, refs); + + EXPECT_EQ(results.size(), 4); // file1, file2, file4, link1 + + // Verify all expected files are in the results + EXPECT_TRUE(results.count(CanonPath("/file1.txt"))); + EXPECT_TRUE(results.count(CanonPath("/file2.txt"))); + EXPECT_TRUE(results.count(CanonPath("/subdir/file4.txt"))); + EXPECT_TRUE(results.count(CanonPath("/link1"))); + EXPECT_FALSE(results.count(CanonPath("/file3.txt"))); + + // Verify the references found in each file are correct + EXPECT_EQ(results.at(CanonPath("/file1.txt")), StorePathSet{path1}); + EXPECT_EQ(results.at(CanonPath("/file2.txt")), StorePathSet({path2, path3})); + EXPECT_EQ(results.at(CanonPath("/subdir/file4.txt")), StorePathSet{path1}); + EXPECT_EQ(results.at(CanonPath("/link1")), StorePathSet{path2}); + } +} + } // namespace nix diff --git a/src/libstore/include/nix/store/path-references.hh b/src/libstore/include/nix/store/path-references.hh index 66d0da268..6aa506da4 100644 --- a/src/libstore/include/nix/store/path-references.hh +++ b/src/libstore/include/nix/store/path-references.hh @@ -3,6 +3,10 @@ #include "nix/store/references.hh" #include "nix/store/path.hh" +#include "nix/util/source-accessor.hh" + +#include +#include namespace nix { @@ -21,4 +25,57 @@ public: StorePathSet getResultPaths(); }; +/** + * Result of scanning a single file for references. + */ +struct FileRefScanResult +{ + CanonPath filePath; ///< The file that was scanned + StorePathSet foundRefs; ///< Which store paths were found in this file +}; + +/** + * Scan a store path tree and report which references appear in which files. + * + * This is like scanForReferences() but provides per-file granularity. + * Useful for cycle detection and detailed dependency analysis like `nix why-depends --precise`. + * + * The function walks the tree using the provided accessor and streams each file's + * contents through a RefScanSink to detect hash references. For each file that + * contains at least one reference, a callback is invoked with the file path and + * the set of references found. + * + * Note: This function only searches for the hash part of store paths (e.g., + * "dc04vv14dak1c1r48qa0m23vr9jy8sm0"), not the name part. A store path like + * "/nix/store/dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo" will be detected if the + * hash appears anywhere in the scanned content, regardless of the "-foo" suffix. + * + * @param accessor Source accessor to read the tree + * @param rootPath Root path to scan + * @param refs Set of store paths to search for + * @param callback Called for each file that contains at least one reference + */ +void scanForReferencesDeep( + SourceAccessor & accessor, + const CanonPath & rootPath, + const StorePathSet & refs, + std::function callback); + +/** + * Scan a store path tree and return which references appear in which files. + * + * This is a convenience wrapper around the callback-based scanForReferencesDeep() + * that collects all results into a map for efficient lookups. + * + * Note: This function only searches for the hash part of store paths, not the name part. + * See the callback-based overload for details. + * + * @param accessor Source accessor to read the tree + * @param rootPath Root path to scan + * @param refs Set of store paths to search for + * @return Map from file paths to the set of references found in each file + */ +std::map +scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs); + } // namespace nix diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 8b167e902..3d783bbe4 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,11 +1,15 @@ #include "nix/store/path-references.hh" #include "nix/util/hash.hh" #include "nix/util/archive.hh" +#include "nix/util/source-accessor.hh" +#include "nix/util/canon-path.hh" +#include "nix/util/logging.hh" #include #include #include #include +#include namespace nix { @@ -54,4 +58,90 @@ StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathS return refsSink.getResultPaths(); } +void scanForReferencesDeep( + SourceAccessor & accessor, + const CanonPath & rootPath, + const StorePathSet & refs, + std::function callback) +{ + // Recursive tree walker + auto walk = [&](this auto & self, const CanonPath & path) -> void { + auto stat = accessor.lstat(path); + + switch (stat.type) { + case SourceAccessor::tRegular: { + // Create a fresh sink for each file to independently detect references. + // RefScanSink accumulates found hashes globally - once a hash is found, + // it remains in the result set. If we reused the same sink across files, + // we couldn't distinguish which files contain which references, as a hash + // found in an earlier file wouldn't be reported when found in later files. + PathRefScanSink sink = PathRefScanSink::fromPaths(refs); + + // Scan this file by streaming its contents through the sink + accessor.readFile(path, sink); + + // Get the references found in this file + auto foundRefs = sink.getResultPaths(); + + // Report if we found anything in this file + if (!foundRefs.empty()) { + debug("scanForReferencesDeep: found %d references in %s", foundRefs.size(), path.abs()); + callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)}); + } + break; + } + + case SourceAccessor::tDirectory: { + // Recursively scan directory contents + auto entries = accessor.readDirectory(path); + for (const auto & [name, entryType] : entries) { + self(path / name); + } + break; + } + + case SourceAccessor::tSymlink: { + // Create a fresh sink for the symlink target (same reason as regular files) + PathRefScanSink sink = PathRefScanSink::fromPaths(refs); + + // Scan symlink target for references + auto target = accessor.readLink(path); + sink(std::string_view(target)); + + // Get the references found in this symlink target + auto foundRefs = sink.getResultPaths(); + + if (!foundRefs.empty()) { + debug("scanForReferencesDeep: found %d references in symlink %s", foundRefs.size(), path.abs()); + callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)}); + } + break; + } + + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: + case SourceAccessor::tUnknown: + default: + throw Error("file '%s' has an unsupported type", path.abs()); + } + }; + + // Start the recursive walk from the root + walk(rootPath); +} + +std::map +scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs) +{ + std::map results; + + scanForReferencesDeep(accessor, rootPath, refs, [&](FileRefScanResult result) { + results[std::move(result.filePath)] = std::move(result.foundRefs); + }); + + return results; +} + } // namespace nix From 6129aee988132742837d36fd4cf995bfe85b3198 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 22:55:14 +0000 Subject: [PATCH 109/201] refactor(nix/why-depends): use scanForReferencesDeep for --precise mode Replaces manual tree-walking and reference scanning with the new scanForReferencesDeep function. --- src/nix/why-depends.cc | 79 +++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index dc30fabd7..29da9e953 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,5 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" +#include "nix/store/path-references.hh" #include "nix/util/source-accessor.hh" #include "nix/main/shared.hh" @@ -191,7 +192,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions /* Sort the references by distance to `dependency` to ensure that the shortest path is printed first. */ std::multimap refs; - StringSet hashes; + StorePathSet refPaths; for (auto & ref : node.refs) { if (ref == node.path && packagePath != dependencyPath) @@ -200,7 +201,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (node2.dist == inf) continue; refs.emplace(node2.dist, &node2); - hashes.insert(std::string(node2.path.hashPart())); + refPaths.insert(node2.path); } /* For each reference, find the files and symlinks that @@ -209,58 +210,50 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions auto accessor = store->requireStoreObjectAccessor(node.path); - auto visitPath = [&](this auto && recur, const CanonPath & p) -> void { - auto st = accessor->maybeLstat(p); - assert(st); + auto getColour = [&](const std::string & hash) { + return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; + }; - auto p2 = p.isRoot() ? p.abs() : p.rel(); + if (precise) { + // Use scanForReferencesDeep to find files containing references + scanForReferencesDeep(*accessor, CanonPath::root, refPaths, [&](FileRefScanResult result) { + auto p2 = result.filePath.isRoot() ? result.filePath.abs() : result.filePath.rel(); + auto st = accessor->lstat(result.filePath); - auto getColour = [&](const std::string & hash) { - return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; - }; + if (st.type == SourceAccessor::Type::tRegular) { + auto contents = accessor->readFile(result.filePath); - if (st->type == SourceAccessor::Type::tDirectory) { - auto names = accessor->readDirectory(p); - for (auto & [name, type] : names) - recur(p / name); - } - - else if (st->type == SourceAccessor::Type::tRegular) { - auto contents = accessor->readFile(p); - - for (auto & hash : hashes) { - auto pos = contents.find(hash); - if (pos != std::string::npos) { - size_t margin = 32; - auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back( - fmt("%s: …%s…", + // For each reference found in this file, extract context + for (auto & foundRef : result.foundRefs) { + std::string hash(foundRef.hashPart()); + auto pos = contents.find(hash); + if (pos != std::string::npos) { + size_t margin = 32; + auto pos2 = pos >= margin ? pos - margin : 0; + hits[hash].emplace_back(fmt( + "%s: …%s…", p2, hilite( filterPrintable(std::string(contents, pos2, pos - pos2 + hash.size() + margin)), pos - pos2, StorePath::HashLen, getColour(hash)))); + } + } + } else if (st.type == SourceAccessor::Type::tSymlink) { + auto target = accessor->readLink(result.filePath); + + // For each reference found in this symlink, show it + for (auto & foundRef : result.foundRefs) { + std::string hash(foundRef.hashPart()); + auto pos = target.find(hash); + if (pos != std::string::npos) + hits[hash].emplace_back( + fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } - } - - else if (st->type == SourceAccessor::Type::tSymlink) { - auto target = accessor->readLink(p); - - for (auto & hash : hashes) { - auto pos = target.find(hash); - if (pos != std::string::npos) - hits[hash].emplace_back( - fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); - } - } - }; - - // FIXME: should use scanForReferences(). - - if (precise) - visitPath(CanonPath::root); + }); + } for (auto & ref : refs) { std::string hash(ref.second->path.hashPart()); From dd716dc9be9d54df959b951d97c51c9eafa37d4d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Oct 2025 15:48:07 -0400 Subject: [PATCH 110/201] Create default `Store::narFromPath` implementation in terms of `getFSAccessor` This is a good default (the methods that allow for an arbitrary choice of source accessor are generally preferable both to implement and to use). And it also pays its way by allowing us to delete *both* the `DummyStore` and `LocalStore` implementations. --- src/libstore/dummy-store.cc | 12 ------------ src/libstore/include/nix/store/local-fs-store.hh | 1 - src/libstore/include/nix/store/store-api.hh | 2 +- src/libstore/include/nix/store/uds-remote-store.hh | 2 +- src/libstore/local-fs-store.cc | 7 ------- src/libstore/restricted-store.cc | 2 +- src/libstore/ssh-store.cc | 2 +- src/libstore/store-api.cc | 7 +++++++ 8 files changed, 11 insertions(+), 24 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 6c8cb3480..1333e0aed 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -258,18 +258,6 @@ struct DummyStoreImpl : DummyStore }); } - void narFromPath(const StorePath & path, Sink & sink) override - { - bool visited = contents.cvisit(path, [&](const auto & kv) { - const auto & [info, accessor] = kv.second; - SourcePath sourcePath(accessor); - dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive); - }); - - if (!visited) - throw Error("path '%s' is not valid", printStorePath(path)); - } - void queryRealisationUncached( const DrvOutput & drvOutput, Callback> callback) noexcept override { diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index 08f8e1656..100a4110d 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -78,7 +78,6 @@ struct LocalFSStore : virtual Store, virtual GcStore, virtual LogStore LocalFSStore(const Config & params); - void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor(bool requireValidPath = true) override; std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) override; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index d03e8e010..8fa13de34 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -609,7 +609,7 @@ public: /** * Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(const StorePath & path, Sink & sink); /** * For each path, if it's a derivation, build it. Building a diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index fe6e486f4..764e8768a 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -68,7 +68,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore void narFromPath(const StorePath & path, Sink & sink) override { - LocalFSStore::narFromPath(path, sink); + Store::narFromPath(path, sink); } /** diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 28069dcaf..1a38cac3b 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -112,13 +112,6 @@ std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & pa return std::make_shared(std::move(absPath)); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) -{ - if (!isValidPath(path)) - throw Error("path '%s' is not valid", printStorePath(path)); - dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); -} - const std::string LocalFSStore::drvsLogDir = "drvs"; std::optional LocalFSStore::getBuildLogExact(const StorePath & path) diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 5270f7d10..ef8aaa380 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -226,7 +226,7 @@ void RestrictedStore::narFromPath(const StorePath & path, Sink & sink) { if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + Store::narFromPath(path, sink); } void RestrictedStore::ensurePath(const StorePath & path) diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index a7e28017f..ce973e734 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -143,7 +143,7 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore void narFromPath(const StorePath & path, Sink & sink) override { - return LocalFSStore::narFromPath(path, sink); + return Store::narFromPath(path, sink); } ref getFSAccessor(bool requireValidPath) override diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cdca6a763..08b75c8fa 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -300,6 +300,13 @@ ValidPathInfo Store::addToStoreSlow( return info; } +void Store::narFromPath(const StorePath & path, Sink & sink) +{ + auto accessor = requireStoreObjectAccessor(path); + SourcePath sourcePath{accessor}; + dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive); +} + StringSet Store::Config::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); From 234f029940ce9bfa86f6f49604a47561400d9e27 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Oct 2025 15:39:58 -0400 Subject: [PATCH 111/201] Add consuming `ref` <-> `std::share_ptr` methods/ctrs This can help churning ref counts when we don't need to. --- src/libutil/include/nix/util/ref.hh | 32 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/libutil/include/nix/util/ref.hh b/src/libutil/include/nix/util/ref.hh index 7cf5ef25e..7ba5349a6 100644 --- a/src/libutil/include/nix/util/ref.hh +++ b/src/libutil/include/nix/util/ref.hh @@ -17,6 +17,12 @@ private: std::shared_ptr p; + void assertNonNull() + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + public: using element_type = T; @@ -24,15 +30,19 @@ public: explicit ref(const std::shared_ptr & p) : p(p) { - if (!p) - throw std::invalid_argument("null pointer cast to ref"); + assertNonNull(); + } + + explicit ref(std::shared_ptr && p) + : p(std::move(p)) + { + assertNonNull(); } explicit ref(T * p) : p(p) { - if (!p) - throw std::invalid_argument("null pointer cast to ref"); + assertNonNull(); } T * operator->() const @@ -45,14 +55,22 @@ public: return *p; } - operator std::shared_ptr() const + std::shared_ptr get_ptr() const & { return p; } - std::shared_ptr get_ptr() const + std::shared_ptr get_ptr() && { - return p; + return std::move(p); + } + + /** + * Convenience to avoid explicit `get_ptr()` call in some cases. + */ + operator std::shared_ptr(this auto && self) + { + return std::forward(self).get_ptr(); } template From 28b73cabccc74304d3474aea8c2d06d4c248f811 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 15:04:56 -0400 Subject: [PATCH 112/201] Make reading and writing derivations store methods This allows for different representations. --- src/libstore/derivations.cc | 27 ++++++++++++++++++--- src/libstore/include/nix/store/store-api.hh | 9 +++++-- src/libstore/store-api.cc | 2 +- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f44bf3e70..20f1d6ca1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -105,7 +105,7 @@ bool BasicDerivation::isBuiltin() const return builder.substr(0, 8) == "builtin:"; } -StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) +static auto infoForDerivation(Store & store, const Derivation & drv) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs.map) @@ -117,13 +117,32 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repa auto contents = drv.unparse(store, false); auto hash = hashString(HashAlgorithm::SHA256, contents); auto ca = TextInfo{.hash = hash, .references = references}; - auto path = store.makeFixedOutputPathFromCA(suffix, ca); + return std::tuple{ + suffix, + contents, + references, + store.makeFixedOutputPathFromCA(suffix, ca), + }; +} - if (readOnly || settings.readOnlyMode || (store.isValidPath(path) && !repair)) +StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) +{ + if (readOnly || settings.readOnlyMode) { + auto [_x, _y, _z, path] = infoForDerivation(store, drv); + return path; + } else + return store.writeDerivation(drv, repair); +} + +StorePath Store::writeDerivation(const Derivation & drv, RepairFlag repair) +{ + auto [suffix, contents, references, path] = infoForDerivation(*this, drv); + + if (isValidPath(path) && !repair) return path; StringSource s{contents}; - auto path2 = store.addToStoreFromDump( + auto path2 = addToStoreFromDump( s, suffix, FileSerialisationMethod::Flat, diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 8fa13de34..522a9a45f 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -778,15 +778,20 @@ public: */ Derivation derivationFromPath(const StorePath & drvPath); + /** + * Write a derivation to the Nix store, and return its path. + */ + virtual StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair); + /** * Read a derivation (which must already be valid). */ - Derivation readDerivation(const StorePath & drvPath); + virtual Derivation readDerivation(const StorePath & drvPath); /** * Read a derivation from a potentially invalid path. */ - Derivation readInvalidDerivation(const StorePath & drvPath); + virtual Derivation readInvalidDerivation(const StorePath & drvPath); /** * @param [out] out Place in here the set of all store paths in the diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 08b75c8fa..c292e2e43 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1170,7 +1170,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) // resolved derivation, so we need to get it first auto resolvedDrv = drv.tryResolve(*this); if (resolvedDrv) - return writeDerivation(*this, *resolvedDrv, NoRepair, true); + return ::nix::writeDerivation(*this, *resolvedDrv, NoRepair, true); } return path; From 136825b4a2700ebbd20f4ba143e9b1819be0537c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 16:00:27 -0400 Subject: [PATCH 113/201] Make Dummy store store derivations separately This makes for more efficiency. Once we have JSON for the dummy store, it will also make for better JSON, too. --- src/libstore-tests/write-derivation.cc | 4 +- src/libstore/dummy-store.cc | 120 ++++++++++++++---- .../include/nix/store/dummy-store-impl.hh | 9 +- 3 files changed, 107 insertions(+), 26 deletions(-) diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index 3f7de05d3..c320f92fa 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -50,8 +50,8 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) EXPECT_EQ(path1, path2); EXPECT_THAT( [&] { writeDerivation(*store, drv, Repair); }, - ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher( - "operation 'addToStoreFromDump' is not supported by store 'dummy://'"))); + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("operation 'writeDerivation' is not supported by store 'dummy://'"))); } } // namespace nix diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 1333e0aed..d11fef73f 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -137,12 +137,31 @@ struct DummyStoreImpl : DummyStore void queryPathInfoUncached( const StorePath & path, Callback> callback) noexcept override { - bool visited = contents.cvisit(path, [&](const auto & kv) { - callback(std::make_shared(StorePath{kv.first}, kv.second.info)); - }); + if (path.isDerivation()) { + if (auto accessor_ = getMemoryFSAccessor(path)) { + ref accessor = ref{std::move(accessor_)}; + /* compute path info on demand */ + auto narHash = + hashPath({accessor, CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + auto info = std::make_shared(path, UnkeyedValidPathInfo{narHash.hash}); + info->narSize = narHash.numBytesDigested; + info->ca = ContentAddress{ + .method = ContentAddressMethod::Raw::Text, + .hash = hashString( + HashAlgorithm::SHA256, + std::get(accessor->root->raw).contents), + }; + callback(std::move(info)); + return; + } + } else { + if (contents.cvisit(path, [&](const auto & kv) { + callback(std::make_shared(StorePath{kv.first}, kv.second.info)); + })) + return; + } - if (!visited) - callback(nullptr); + callback(nullptr); } /** @@ -169,18 +188,25 @@ struct DummyStoreImpl : DummyStore if (checkSigs) throw Error("checking signatures is not supported for '%s' store", config->getHumanReadableURI()); - auto temp = make_ref(); - MemorySink tempSink{*temp}; + auto accessor = make_ref(); + MemorySink tempSink{*accessor}; parseDump(tempSink, source); auto path = info.path; - auto accessor = make_ref(std::move(*temp)); - contents.insert( - {path, - PathInfoAndContents{ - std::move(info), - accessor, - }}); + if (info.path.isDerivation()) { + warn("back compat supporting `addToStore` for inserting derivations in dummy store"); + writeDerivation( + parseDerivation(*this, accessor->readFile(CanonPath::root), Derivation::nameFromPath(info.path))); + return; + } + + contents.insert({ + path, + PathInfoAndContents{ + std::move(info), + accessor, + }, + }); wholeStoreView->addObject(path.to_string(), accessor); } @@ -193,6 +219,9 @@ struct DummyStoreImpl : DummyStore const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override { + if (isDerivation(name)) + throw Error("Do not insert derivation into dummy store with `addToStoreFromDump`"); + if (config->readOnly) unsupported("addToStoreFromDump"); @@ -239,17 +268,47 @@ struct DummyStoreImpl : DummyStore auto path = info.path; auto accessor = make_ref(std::move(*temp)); - contents.insert( - {path, - PathInfoAndContents{ - std::move(info), - accessor, - }}); + contents.insert({ + path, + PathInfoAndContents{ + std::move(info), + accessor, + }, + }); wholeStoreView->addObject(path.to_string(), accessor); return path; } + StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair) override + { + auto drvPath = ::nix::writeDerivation(*this, drv, repair, /*readonly=*/true); + + if (!derivations.contains(drvPath) || repair) { + if (config->readOnly) + unsupported("writeDerivation"); + derivations.insert({drvPath, drv}); + } + + return drvPath; + } + + Derivation readDerivation(const StorePath & drvPath) override + { + if (std::optional res = getConcurrent(derivations, drvPath)) + return *res; + else + throw Error("derivation '%s' is not valid", printStorePath(drvPath)); + } + + /** + * No such thing as an "invalid derivation" with the dummy store + */ + Derivation readInvalidDerivation(const StorePath & drvPath) override + { + return readDerivation(drvPath); + } + void registerDrvOutput(const Realisation & output) override { auto ref = make_ref(output); @@ -273,13 +332,28 @@ struct DummyStoreImpl : DummyStore callback(nullptr); } - std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath) override + std::shared_ptr getMemoryFSAccessor(const StorePath & path, bool requireValidPath = true) { - std::shared_ptr res; - contents.cvisit(path, [&](const auto & kv) { res = kv.second.contents.get_ptr(); }); + std::shared_ptr res; + if (path.isDerivation()) + derivations.cvisit(path, [&](const auto & kv) { + /* compute path info on demand */ + auto res2 = make_ref(); + res2->root = MemorySourceAccessor::File::Regular{ + .contents = kv.second.unparse(*this, false), + }; + res = std::move(res2).get_ptr(); + }); + else + contents.cvisit(path, [&](const auto & kv) { res = kv.second.contents.get_ptr(); }); return res; } + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) override + { + return getMemoryFSAccessor(path, requireValidPath); + } + ref getFSAccessor(bool requireValidPath) override { return wholeStoreView; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 4c9f54e98..137f81c9b 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/dummy-store.hh" +#include "nix/store/derivations.hh" #include @@ -25,11 +26,17 @@ struct DummyStore : virtual Store }; /** - * This is map conceptually owns the file system objects for each + * This map conceptually owns the file system objects for each * store object. */ boost::concurrent_flat_map contents; + /** + * This map conceptually owns every derivation, allowing us to + * avoid "on-disk drv format" serialization round-trips. + */ + boost::concurrent_flat_map derivations; + /** * The build trace maps the pair of a content-addressing (fixed or * floating) derivations an one of its output to a From 18941a2421f40efad264ad1fc4e07339075491c3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Oct 2025 11:37:14 -0400 Subject: [PATCH 114/201] Optimize `DummyStore::isValidPathUncached` See the API docs for the rationale of why this is needed. --- src/libstore/dummy-store.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index d11fef73f..c45a13cc3 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -164,6 +164,15 @@ struct DummyStoreImpl : DummyStore callback(nullptr); } + /** + * Do this to avoid `queryPathInfoUncached` computing `PathInfo` + * that we don't need just to return a `bool`. + */ + bool isValidPathUncached(const StorePath & path) override + { + return path.isDerivation() ? derivations.contains(path) : Store::isValidPathUncached(path); + } + /** * The dummy store is incapable of *not* trusting! :) */ From ad664ce64e90234e6a0349b7b14f00bc9c82bf8e Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Mon, 27 Oct 2025 20:56:54 +0000 Subject: [PATCH 115/201] ci: cancel previous workflow runs on PR updates Add concurrency group configuration to the CI workflow to automatically cancel outdated runs when a PR receives new commits or is force-pushed. This prevents wasting CI resources on superseded code. --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a0820903..67e97b188 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,10 @@ on: default: true type: boolean +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + permissions: read-all jobs: From 4b6d07d64299e539ba4f421a6589abc4e630c36f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:53:39 +0000 Subject: [PATCH 116/201] feat(libstore/s3-binary-cache-store): implement `createMultipartUpload()` POST to key with `?uploads` query parameter, optionally set `Content-Encoding` header, parse `uploadId` from XML response using regex --- src/libstore/s3-binary-cache-store.cc | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 98f742c70..58cb72776 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -4,6 +4,7 @@ #include #include +#include namespace nix { @@ -27,6 +28,15 @@ public: private: ref s3Config; + /** + * Creates a multipart upload for large objects to S3. + * + * @see + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_RequestSyntax + */ + std::string createMultipartUpload( + std::string_view key, std::string_view mimeType, std::optional contentEncoding); + /** * Abort a multipart upload * @@ -45,6 +55,39 @@ void S3BinaryCacheStore::upsertFile( HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint); } +std::string S3BinaryCacheStore::createMultipartUpload( + std::string_view key, std::string_view mimeType, std::optional contentEncoding) +{ + auto req = makeRequest(key); + + // setupForS3() converts s3:// to https:// but strips query parameters + // So we call it first, then add our multipart parameters + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploads"] = ""; + req.uri = VerbatimURL(url); + + req.method = HttpMethod::POST; + req.data = ""; + req.mimeType = mimeType; + + if (contentEncoding) { + req.headers.emplace_back("Content-Encoding", *contentEncoding); + } + + auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + + std::regex uploadIdRegex("([^<]+)"); + std::smatch match; + + if (std::regex_search(result.data, match, uploadIdRegex)) { + return match[1]; + } + + throw Error("S3 CreateMultipartUpload response missing "); +} + void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) { auto req = makeRequest(key); From c592090fffde2fc107dec0bfd398ae7a9c0b4f35 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 08:02:25 +0000 Subject: [PATCH 117/201] feat(libstore/s3-binary-cache-store): implement `uploadPart()` Implement `uploadPart()` for uploading individual parts in S3 multipart uploads: - Constructs URL with `?partNumber=N&uploadId=ID` query parameters - Uploads chunk data with `application/octet-stream` mime type - Extracts and returns `ETag` from response --- src/libstore/s3-binary-cache-store.cc | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 58cb72776..828e75b7c 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -37,6 +37,15 @@ private: std::string createMultipartUpload( std::string_view key, std::string_view mimeType, std::optional contentEncoding); + /** + * Uploads a single part of a multipart upload + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html#API_UploadPart_RequestSyntax + * + * @returns the [ETag](https://en.wikipedia.org/wiki/HTTP_ETag) + */ + std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data); + /** * Abort a multipart upload * @@ -88,6 +97,28 @@ std::string S3BinaryCacheStore::createMultipartUpload( throw Error("S3 CreateMultipartUpload response missing "); } +std::string +S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["partNumber"] = std::to_string(partNumber); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.data = std::move(data); + req.mimeType = "application/octet-stream"; + + auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + + if (result.etag.empty()) { + throw Error("S3 UploadPart response missing ETag for part %d", partNumber); + } + + return std::move(result.etag); +} + void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) { auto req = makeRequest(key); From 3775a2a2268bbc18716363e38868e3bf76fd3884 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:22:54 +0000 Subject: [PATCH 118/201] build(deps): bump actions/upload-artifact from 4 to 5 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 5. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..18ae4d8bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,13 +125,13 @@ jobs: cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY if: ${{ matrix.instrumented }} - name: Upload coverage reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: coverage-reports path: coverage-reports/ if: ${{ matrix.instrumented }} - name: Upload installer tarball - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: installer-${{matrix.os}} path: out/* From ccc06451df3ca9345977ca4cdf7d412f6603dd90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:35:42 +0000 Subject: [PATCH 119/201] build(deps): bump actions/download-artifact from 5 to 6 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 5 to 6. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..10103847a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Download installer tarball - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: installer-${{matrix.os}} path: out From c77317b1a9086b9aa8ff1b22da051e520febe871 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:54:49 +0000 Subject: [PATCH 120/201] feat(libstore/s3-binary-cache-store): implement `completeMultipartUpload()` `completeMultipartUpload()`: Build XML with part numbers and `ETags`, POST to key with `?uploadId` to finalize the multipart upload --- src/libstore/s3-binary-cache-store.cc | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 828e75b7c..178373778 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -5,6 +5,7 @@ #include #include #include +#include namespace nix { @@ -46,6 +47,19 @@ private: */ std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data); + struct UploadedPart + { + uint64_t partNumber; + std::string etag; + }; + + /** + * Completes a multipart upload by combining all uploaded parts. + * @see + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_RequestSyntax + */ + void completeMultipartUpload(std::string_view key, std::string_view uploadId, std::span parts); + /** * Abort a multipart upload * @@ -132,6 +146,34 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_ getFileTransfer()->enqueueFileTransfer(req).get(); } +void S3BinaryCacheStore::completeMultipartUpload( + std::string_view key, std::string_view uploadId, std::span parts) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::POST; + + std::string xml = ""; + for (const auto & part : parts) { + xml += ""; + xml += "" + std::to_string(part.partNumber) + ""; + xml += "" + part.etag + ""; + xml += ""; + } + xml += ""; + + debug("S3 CompleteMultipartUpload XML (%d parts): %s", parts.size(), xml); + + req.data = xml; + req.mimeType = "text/xml"; + + getFileTransfer()->enqueueFileTransfer(req).get(); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"}; From 94965a3a3eeac6574a06a36760e6470977a7c1f9 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 20:15:25 +0000 Subject: [PATCH 121/201] test(nixos): add S3 multipart upload integration tests --- tests/nixos/s3-binary-cache-store.nix | 129 ++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index a2ede4572..a07375489 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -34,8 +34,10 @@ in pkgA pkgB pkgC + pkgs.coreutils ]; environment.systemPackages = [ pkgs.minio-client ]; + nix.nixPath = [ "nixpkgs=${pkgs.path}" ]; nix.extraOptions = '' experimental-features = nix-command substituters = @@ -639,6 +641,129 @@ in ) print(" ✓ Fetch with versionId parameter works") + @setup_s3() + def test_multipart_upload_basic(bucket): + """Test basic multipart upload with a large file""" + print("\n--- Test: Multipart Upload Basic ---") + + large_file_size = 10 * 1024 * 1024 + large_pkg = server.succeed( + "nix-store --add $(dd if=/dev/urandom of=/tmp/large-file bs=1M count=10 2>/dev/null && echo /tmp/large-file)" + ).strip() + + chunk_size = 5 * 1024 * 1024 + expected_parts = 3 # 10 MB raw becomes ~10.5 MB compressed (NAR + xz overhead) + + store_url = make_s3_url( + bucket, + **{ + "multipart-upload": "true", + "multipart-threshold": str(5 * 1024 * 1024), + "multipart-chunk-size": str(chunk_size), + } + ) + + print(f" Uploading {large_file_size} byte file (expect {expected_parts} parts)") + output = server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {large_pkg} --debug 2>&1") + + if "using S3 multipart upload" not in output: + raise Exception("Expected multipart upload to be used") + + expected_msg = f"{expected_parts} parts uploaded" + if expected_msg not in output: + print("Debug output:") + print(output) + raise Exception(f"Expected '{expected_msg}' in output") + + print(f" ✓ Multipart upload used with {expected_parts} parts") + + client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' {large_pkg} --no-check-sigs") + verify_packages_in_store(client, large_pkg, should_exist=True) + + print(" ✓ Large file downloaded and verified") + + @setup_s3() + def test_multipart_threshold(bucket): + """Test that files below threshold use regular upload""" + print("\n--- Test: Multipart Threshold Behavior ---") + + store_url = make_s3_url( + bucket, + **{ + "multipart-upload": "true", + "multipart-threshold": str(1024 * 1024 * 1024), + } + ) + + print(" Uploading small file with high threshold") + output = server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']} --debug 2>&1") + + if "using S3 multipart upload" in output: + raise Exception("Should not use multipart for file below threshold") + + if "using S3 regular upload" not in output: + raise Exception("Expected regular upload to be used") + + print(" ✓ Regular upload used for file below threshold") + + client.succeed(f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}") + verify_packages_in_store(client, PKGS['A'], should_exist=True) + + print(" ✓ Small file uploaded and verified") + + @setup_s3() + def test_multipart_with_log_compression(bucket): + """Test multipart upload with compressed build logs""" + print("\n--- Test: Multipart Upload with Log Compression ---") + + # Create a derivation that produces a large text log (12 MB of base64 output) + drv_path = server.succeed( + """ + nix-instantiate --expr ' + let pkgs = import {}; + in derivation { + name = "large-log-builder"; + builder = "/bin/sh"; + args = ["-c" "$coreutils/bin/dd if=/dev/urandom bs=1M count=12 | $coreutils/bin/base64; echo success > $out"]; + coreutils = pkgs.coreutils; + system = builtins.currentSystem; + } + ' + """ + ).strip() + + print(" Building derivation to generate large log") + server.succeed(f"nix-store --realize {drv_path} &>/dev/null") + + # Upload logs with compression and multipart + store_url = make_s3_url( + bucket, + **{ + "multipart-upload": "true", + "multipart-threshold": str(5 * 1024 * 1024), + "multipart-chunk-size": str(5 * 1024 * 1024), + "log-compression": "xz", + } + ) + + print(" Uploading build log with compression and multipart") + output = server.succeed( + f"{ENV_WITH_CREDS} nix store copy-log --to '{store_url}' {drv_path} --debug 2>&1" + ) + + # Should use multipart for the compressed log + if "using S3 multipart upload" not in output or "log/" not in output: + print("Debug output:") + print(output) + raise Exception("Expected multipart upload to be used for compressed log") + + if "parts uploaded" not in output: + print("Debug output:") + print(output) + raise Exception("Expected multipart completion message") + + print(" ✓ Compressed log uploaded with multipart") + # ============================================================================ # Main Test Execution # ============================================================================ @@ -669,6 +794,10 @@ in test_compression_disabled() test_nix_prefetch_url() test_versioned_urls() + # FIXME: enable when multipart fully lands + # test_multipart_upload_basic() + # test_multipart_threshold() + # test_multipart_with_log_compression() print("\n" + "="*80) print("✓ All S3 Binary Cache Store Tests Passed!") From 972915cabd772c4056fc4d08abd0579f1c252147 Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Tue, 28 Oct 2025 09:36:46 +0000 Subject: [PATCH 122/201] docs: remove incorrect claim re gc --print-dead Per #7591, the `nix-store --gc --print-dead` command does not provide any feedback about the amount of disk space that is used by dead store paths. It looks like this has been the case since 7ab68961e (* Garbage collector: added an option `--use-atime' to delete paths in..., 2008-09-17). Update the nix-store documentation to remove the claim that this is function that `nix-store --gc --print-dead` performs. --- doc/manual/source/command-ref/nix-store/gc.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/source/command-ref/nix-store/gc.md b/doc/manual/source/command-ref/nix-store/gc.md index f432e00eb..8ec59d906 100644 --- a/doc/manual/source/command-ref/nix-store/gc.md +++ b/doc/manual/source/command-ref/nix-store/gc.md @@ -48,8 +48,7 @@ The behaviour of the collector is also influenced by the configuration file. By default, the collector prints the total number of freed bytes when it -finishes (or when it is interrupted). With `--print-dead`, it prints the -number of bytes that would be freed. +finishes (or when it is interrupted). {{#include ./opt-common.md}} From 5fc0c4f1027f673f76768b2e8659321cedda6834 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 22 Sep 2025 14:07:03 +0200 Subject: [PATCH 123/201] doc: Improve libexpr-c docs - Uses the more explicit `@ingroup` most of the time, to avoid problems with nested groups, and to make group membership more explicit. The division into headers is not great for documentation purposes, so this helps. - More attention for memory management details - Various other improvements to doc comments --- .../source/development/documentation.md | 6 + src/external-api-docs/README.md | 2 +- src/libexpr-c/nix_api_expr.h | 43 ++- src/libexpr-c/nix_api_external.h | 12 +- src/libexpr-c/nix_api_value.h | 250 +++++++++++++----- src/libutil-c/nix_api_util.h | 2 + 6 files changed, 237 insertions(+), 78 deletions(-) diff --git a/doc/manual/source/development/documentation.md b/doc/manual/source/development/documentation.md index a2a54175d..6823780cc 100644 --- a/doc/manual/source/development/documentation.md +++ b/doc/manual/source/development/documentation.md @@ -240,3 +240,9 @@ $ configurePhase $ ninja src/external-api-docs/html $ xdg-open src/external-api-docs/html/index.html ``` + +If you use direnv, or otherwise want to run `configurePhase` in a transient shell, use: + +```bash +nix-shell -A devShells.x86_64-linux.native-clangStdenv --command 'mesonFlags="$mesonFlags -Ddoc-gen=true"; mesonConfigurePhase' +``` diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md index 8760ac88b..1940cc1c0 100644 --- a/src/external-api-docs/README.md +++ b/src/external-api-docs/README.md @@ -15,7 +15,7 @@ programmatically: 1. Embedding the evaluator 2. Writing language plug-ins -Embedding means you link the Nix C libraries in your program and use them from +Embedding means you link the Nix C API libraries in your program and use them from there. Adding a plug-in means you make a library that gets loaded by the Nix language evaluator, specified through a configuration option. diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 2be739955..3623ee076 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -4,11 +4,14 @@ * @brief Bindings to the Nix language evaluator * * See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example. - * @{ */ /** @file * @brief Main entry for the libexpr C bindings */ +/** @defgroup libexpr_init Initialization + * @ingroup libexpr + * @{ + */ #include "nix_api_store.h" #include "nix_api_util.h" @@ -45,7 +48,10 @@ typedef struct nix_eval_state_builder nix_eval_state_builder; */ typedef struct EvalState EvalState; // nix::EvalState +/** @} */ + /** @brief A Nix language value, or thunk that may evaluate to a value. + * @ingroup value * * Values are the primary objects manipulated in the Nix language. * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its @@ -56,7 +62,8 @@ typedef struct EvalState EvalState; // nix::EvalState * * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules. * - * @see value_manip + * @struct nix_value + * @see value_create, value_extract * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; @@ -65,6 +72,7 @@ NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** * @brief Initialize the Nix language evaluator. + * @ingroup libexpr_init * * This function must be called at least once, * at some point before constructing a EvalState for the first time. @@ -77,6 +85,7 @@ nix_err nix_libexpr_init(nix_c_context * context); /** * @brief Parses and evaluates a Nix expression from a string. + * @ingroup value_create * * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. @@ -93,6 +102,7 @@ nix_err nix_expr_eval_from_string( /** * @brief Calls a Nix function with an argument. + * @ingroup value_create * * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. @@ -107,6 +117,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * f /** * @brief Calls a Nix function with multiple arguments. + * @ingroup value_create * * Technically these are functions that return functions. It is common for Nix * functions to be curried, so this function is useful for calling them. @@ -126,10 +137,12 @@ nix_err nix_value_call_multi( /** * @brief Calls a Nix function with multiple arguments. + * @ingroup value_create * * Technically these are functions that return functions. It is common for Nix * functions to be curried, so this function is useful for calling them. * + * @def NIX_VALUE_CALL * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. * @param[out] value The result of the function call. @@ -147,6 +160,7 @@ nix_err nix_value_call_multi( /** * @brief Forces the evaluation of a Nix value. + * @ingroup value_create * * The Nix interpreter is lazy, and not-yet-evaluated values can be * of type NIX_TYPE_THUNK instead of their actual value. @@ -180,18 +194,20 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val /** * @brief Create a new nix_eval_state_builder + * @ingroup libexpr_init * * The settings are initialized to their default value. * Values can be sourced elsewhere with nix_eval_state_builder_load. * * @param[out] context Optional, stores error information * @param[in] store The Nix store to use. - * @return A new nix_eval_state_builder or NULL on failure. + * @return A new nix_eval_state_builder or NULL on failure. Call nix_eval_state_builder_free() when you're done. */ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store); /** * @brief Read settings from the ambient environment + * @ingroup libexpr_init * * Settings are sourced from environment variables and configuration files, * as documented in the Nix manual. @@ -204,6 +220,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil /** * @brief Set the lookup path for `<...>` expressions + * @ingroup libexpr_init * * @param[in] context Optional, stores error information * @param[in] builder The builder to modify. @@ -214,18 +231,21 @@ nix_err nix_eval_state_builder_set_lookup_path( /** * @brief Create a new Nix language evaluator state + * @ingroup libexpr_init * - * Remember to nix_eval_state_builder_free after building the state. + * The builder becomes unusable after this call. Remember to call nix_eval_state_builder_free() + * after building the state. * * @param[out] context Optional, stores error information * @param[in] builder The builder to use and free - * @return A new Nix state or NULL on failure. + * @return A new Nix state or NULL on failure. Call nix_state_free() when you're done. * @see nix_eval_state_builder_new, nix_eval_state_builder_free */ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder); /** * @brief Free a nix_eval_state_builder + * @ingroup libexpr_init * * Does not fail. * @@ -235,19 +255,21 @@ void nix_eval_state_builder_free(nix_eval_state_builder * builder); /** * @brief Create a new Nix language evaluator state + * @ingroup libexpr_init * * For more control, use nix_eval_state_builder * * @param[out] context Optional, stores error information * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. * @param[in] store The Nix store to use. - * @return A new Nix state or NULL on failure. + * @return A new Nix state or NULL on failure. Call nix_state_free() when you're done. * @see nix_state_builder_new */ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store); /** * @brief Frees a Nix state. + * @ingroup libexpr_init * * Does not fail. * @@ -256,6 +278,7 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, void nix_state_free(EvalState * state); /** @addtogroup GC + * @ingroup libexpr * @brief Reference counting and garbage collector operations * * The Nix language evaluator uses a garbage collector. To ease C interop, we implement @@ -286,6 +309,9 @@ nix_err nix_gc_incref(nix_c_context * context, const void * object); /** * @brief Decrement the garbage collector reference counter for the given object * + * @deprecated We are phasing out the general nix_gc_decref() in favor of type-specified free functions, such as + * nix_value_decref(). + * * We also provide typed `nix_*_decref` functions, which are * - safer to use * - easier to integrate when deriving bindings @@ -314,12 +340,11 @@ void nix_gc_now(); */ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)); -/** @} */ +/** @} */ // doxygen group GC + // cffi end #ifdef __cplusplus } #endif -/** @} */ - #endif // NIX_API_EXPR_H diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h index f4a327281..96c479d57 100644 --- a/src/libexpr-c/nix_api_external.h +++ b/src/libexpr-c/nix_api_external.h @@ -2,11 +2,12 @@ #define NIX_API_EXTERNAL_H /** @ingroup libexpr * @addtogroup Externals - * @brief Deal with external values + * @brief Externals let Nix expressions work with foreign values that aren't part of the normal Nix value data model * @{ */ /** @file * @brief libexpr C bindings dealing with external values + * @see Externals */ #include "nix_api_expr.h" @@ -115,7 +116,7 @@ typedef struct NixCExternalValueDesc * @brief Try to compare two external values * * Optional, the default is always false. - * If the other object was not a Nix C external value, this comparison will + * If the other object was not a Nix C API external value, this comparison will * also return false * @param[in] self the void* passed to nix_create_external_value * @param[in] other the void* passed to the other object's @@ -168,7 +169,7 @@ typedef struct NixCExternalValueDesc /** * @brief Create an external value, that can be given to nix_init_external * - * Owned by the GC. Use nix_gc_decref when you're done with the pointer. + * Call nix_gc_decref() when you're done with the pointer. * * @param[out] context Optional, stores error information * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long @@ -180,10 +181,11 @@ typedef struct NixCExternalValueDesc ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v); /** - * @brief Extract the pointer from a nix c external value. + * @brief Extract the pointer from a Nix C API external value. * @param[out] context Optional, stores error information * @param[in] b The external value - * @returns The pointer, or null if the external value was not from nix c. + * @returns The pointer, valid while the external value is valid, or null if the external value was not from the Nix C + * API. * @see nix_get_external */ void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b); diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 835eaec6e..5bd45da90 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -1,9 +1,6 @@ #ifndef NIX_API_VALUE_H #define NIX_API_VALUE_H -/** @addtogroup libexpr - * @{ - */ /** @file * @brief libexpr C bindings dealing with values */ @@ -20,18 +17,89 @@ extern "C" { #endif // cffi start +/** @defgroup value Value + * @ingroup libexpr + * @brief nix_value type and core operations for working with Nix values + * @see value_create + * @see value_extract + */ + +/** @defgroup value_create Value Creation + * @ingroup libexpr + * @brief Functions for allocating and initializing Nix values + * + * Values are usually created with `nix_alloc_value` followed by `nix_init_*` functions. + * In primop callbacks, allocation is already done and only initialization is needed. + */ + +/** @defgroup value_extract Value Extraction + * @ingroup libexpr + * @brief Functions for extracting data from Nix values + */ + +/** @defgroup primops PrimOps and Builtins + * @ingroup libexpr + */ + // Type definitions +/** @brief Represents the state of a Nix value + * + * Thunk values (NIX_TYPE_THUNK) change to their final, unchanging type when forced. + * + * @see https://nix.dev/manual/nix/latest/language/evaluation.html + * @enum ValueType + * @ingroup value + */ typedef enum { + /** Unevaluated expression + * + * Thunks often contain an expression and closure, but may contain other + * representations too. + * + * Their state is mutable, unlike that of the other types. + */ NIX_TYPE_THUNK, + /** + * A 64 bit signed integer. + */ NIX_TYPE_INT, + /** @brief IEEE 754 double precision floating point number + * @see https://nix.dev/manual/nix/latest/language/types.html#type-float + */ NIX_TYPE_FLOAT, + /** @brief Boolean true or false value + * @see https://nix.dev/manual/nix/latest/language/types.html#type-bool + */ NIX_TYPE_BOOL, + /** @brief String value with context + * + * String content may contain arbitrary bytes, not necessarily UTF-8. + * @see https://nix.dev/manual/nix/latest/language/types.html#type-string + */ NIX_TYPE_STRING, + /** @brief Filesystem path + * @see https://nix.dev/manual/nix/latest/language/types.html#type-path + */ NIX_TYPE_PATH, + /** @brief Null value + * @see https://nix.dev/manual/nix/latest/language/types.html#type-null + */ NIX_TYPE_NULL, + /** @brief Attribute set (key-value mapping) + * @see https://nix.dev/manual/nix/latest/language/types.html#type-attrs + */ NIX_TYPE_ATTRS, + /** @brief Ordered list of values + * @see https://nix.dev/manual/nix/latest/language/types.html#type-list + */ NIX_TYPE_LIST, + /** @brief Function (lambda or builtin) + * @see https://nix.dev/manual/nix/latest/language/types.html#type-function + */ NIX_TYPE_FUNCTION, + /** @brief External value from C++ plugins or C API + * @see Externals + */ NIX_TYPE_EXTERNAL } ValueType; @@ -39,22 +107,41 @@ typedef enum { typedef struct nix_value nix_value; typedef struct EvalState EvalState; +/** @deprecated Use nix_value instead */ [[deprecated("use nix_value instead")]] typedef nix_value Value; // type defs /** @brief Stores an under-construction set of bindings - * @ingroup value_manip + * @ingroup value_create * - * Do not reuse. + * Each builder can only be used once. After calling nix_make_attrs(), the builder + * becomes invalid and must not be used again. Call nix_bindings_builder_free() to release it. + * + * Typical usage pattern: + * 1. Create with nix_make_bindings_builder() + * 2. Insert attributes with nix_bindings_builder_insert() + * 3. Create final attribute set with nix_make_attrs() + * 4. Free builder with nix_bindings_builder_free() + * + * @struct BindingsBuilder * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs * @see nix_bindings_builder_insert */ typedef struct BindingsBuilder BindingsBuilder; /** @brief Stores an under-construction list - * @ingroup value_manip + * @ingroup value_create * - * Do not reuse. + * Each builder can only be used once. After calling nix_make_list(), the builder + * becomes invalid and must not be used again. Call nix_list_builder_free() to release it. + * + * Typical usage pattern: + * 1. Create with nix_make_list_builder() + * 2. Insert elements with nix_list_builder_insert() + * 3. Create final list with nix_make_list() + * 4. Free builder with nix_list_builder_free() + * + * @struct ListBuilder * @see nix_make_list_builder, nix_list_builder_free, nix_make_list * @see nix_list_builder_insert */ @@ -63,25 +150,28 @@ typedef struct ListBuilder ListBuilder; /** @brief PrimOp function * @ingroup primops * - * Owned by the GC - * @see nix_alloc_primop, nix_init_primop + * Can be released with nix_gc_decref() when necessary. + * @struct PrimOp + * @see nix_alloc_primop, nix_init_primop, nix_register_primop */ typedef struct PrimOp PrimOp; /** @brief External Value * @ingroup Externals * - * Owned by the GC + * Can be released with nix_gc_decref() when necessary. + * @struct ExternalValue + * @see nix_create_external_value, nix_init_external, nix_get_external */ typedef struct ExternalValue ExternalValue; /** @brief String without placeholders, and realised store paths + * @struct nix_realised_string + * @see nix_string_realise, nix_realised_string_free */ typedef struct nix_realised_string nix_realised_string; -/** @defgroup primops Adding primops - * @{ - */ /** @brief Function pointer for primops + * @ingroup primops * * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here"). * @@ -97,9 +187,9 @@ typedef void (*PrimOpFun)( void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret); /** @brief Allocate a PrimOp + * @ingroup primops * - * Owned by the garbage collector. - * Use nix_gc_decref() when you're done with the returned PrimOp. + * Call nix_gc_decref() when you're done with the returned PrimOp. * * @param[out] context Optional, stores error information * @param[in] fun callback @@ -121,35 +211,38 @@ PrimOp * nix_alloc_primop( void * user_data); /** @brief add a primop to the `builtins` attribute set + * @ingroup primops * * Only applies to States created after this call. * - * Moves your PrimOp content into the global evaluator - * registry, meaning your input PrimOp pointer is no longer usable. - * You are free to remove your references to it, - * after which it will be garbage collected. + * Moves your PrimOp content into the global evaluator registry, meaning + * your input PrimOp pointer becomes invalid. The PrimOp must not be used + * with nix_init_primop() before or after this call, as this would cause + * undefined behavior. + * You must call nix_gc_decref() on the original PrimOp pointer + * after this call to release your reference. * * @param[out] context Optional, stores error information - * @return primop, or null in case of errors - * + * @param[in] primOp PrimOp to register + * @return error code, NIX_OK on success */ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); -/** @} */ // Function prototypes /** @brief Allocate a Nix value + * @ingroup value_create * - * Owned by the GC. Use nix_gc_decref() when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] state nix evaluator state * @return value, or null in case of errors - * */ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state); /** * @brief Increment the garbage collector reference counter for the given `nix_value`. + * @ingroup value * * The Nix language evaluator C API keeps track of alive objects by reference counting. * When you're done with a refcounted pointer, call nix_value_decref(). @@ -161,21 +254,19 @@ nix_err nix_value_incref(nix_c_context * context, nix_value * value); /** * @brief Decrement the garbage collector reference counter for the given object + * @ingroup value + * + * When the counter reaches zero, the `nix_value` object becomes invalid. + * The data referenced by `nix_value` may not be deallocated until the memory + * garbage collector has run, but deallocation is not guaranteed. * * @param[out] context Optional, stores error information * @param[in] value The object to stop referencing */ nix_err nix_value_decref(nix_c_context * context, nix_value * value); -/** @addtogroup value_manip Manipulating values - * @brief Functions to inspect and change Nix language values, represented by nix_value. - * @{ - */ -/** @anchor getters - * @name Getters - */ -/**@{*/ /** @brief Get value type + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return type of nix value @@ -183,14 +274,15 @@ nix_err nix_value_decref(nix_c_context * context, nix_value * value); ValueType nix_get_type(nix_c_context * context, const nix_value * value); /** @brief Get type name of value as defined in the evaluator + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return type name, owned string - * @todo way to free the result + * @return type name string, free with free() */ const char * nix_get_typename(nix_c_context * context, const nix_value * value); /** @brief Get boolean value + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return true or false, error info via context @@ -198,6 +290,7 @@ const char * nix_get_typename(nix_c_context * context, const nix_value * value); bool nix_get_bool(nix_c_context * context, const nix_value * value); /** @brief Get the raw string + * @ingroup value_extract * * This may contain placeholders. * @@ -205,21 +298,21 @@ bool nix_get_bool(nix_c_context * context, const nix_value * value); * @param[in] value Nix value to inspect * @param[in] callback Called with the string value. * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. - * @return string * @return error code, NIX_OK on success. */ nix_err nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data); /** @brief Get path as string + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return string, if the type is NIX_TYPE_PATH - * @return NULL in case of error. + * @return string valid while value is valid, NULL in case of error */ const char * nix_get_path_string(nix_c_context * context, const nix_value * value); /** @brief Get the length of a list + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return length of list, error info via context @@ -227,6 +320,7 @@ const char * nix_get_path_string(nix_c_context * context, const nix_value * valu unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value); /** @brief Get the element count of an attrset + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return attrset element count, error info via context @@ -234,6 +328,7 @@ unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value) unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value); /** @brief Get float value in 64 bits + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return float contents, error info via context @@ -241,6 +336,7 @@ unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value double nix_get_float(nix_c_context * context, const nix_value * value); /** @brief Get int value + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return int contents, error info via context @@ -248,15 +344,18 @@ double nix_get_float(nix_c_context * context, const nix_value * value); int64_t nix_get_int(nix_c_context * context, const nix_value * value); /** @brief Get external reference + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return reference to external, NULL in case of error + * @return reference valid while value is valid. Call nix_gc_incref() if you need it to live longer, then only in that + * case call nix_gc_decref() when done. NULL in case of error */ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value); /** @brief Get the ix'th element of a list + * @ingroup value_extract * - * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state @@ -266,11 +365,12 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value); nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); /** @brief Get the ix'th element of a list without forcing evaluation of the element + * @ingroup value_extract * * Returns the list element without forcing its evaluation, allowing access to lazy values. * The list value itself must already be evaluated. * - * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect (must be an evaluated list) * @param[in] state nix evaluator state @@ -281,8 +381,9 @@ nix_value * nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); /** @brief Get an attr by name + * @ingroup value_extract * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state @@ -292,11 +393,12 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Get an attribute value by attribute name, without forcing evaluation of the attribute's value + * @ingroup value_extract * * Returns the attribute value without forcing its evaluation, allowing access to lazy values. * The attribute set value itself must already be evaluated. * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect (must be an evaluated attribute set) * @param[in] state nix evaluator state @@ -307,6 +409,7 @@ nix_value * nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Check if an attribute name exists on a value + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state @@ -316,6 +419,7 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Get an attribute by index + * @ingroup value_extract * * Also gives you the name. * @@ -329,18 +433,19 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS * lexicographic order by Unicode scalar value for valid UTF-8). We recommend * applying this same ordering for consistency. * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state * @param[in] i attribute index - * @param[out] name will store a pointer to the attribute name + * @param[out] name will store a pointer to the attribute name, valid until state is freed * @return value, NULL in case of errors */ nix_value * nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name); /** @brief Get an attribute by index, without forcing evaluation of the attribute's value + * @ingroup value_extract * * Also gives you the name. * @@ -357,18 +462,19 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state * lexicographic order by Unicode scalar value for valid UTF-8). We recommend * applying this same ordering for consistency. * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect (must be an evaluated attribute set) * @param[in] state nix evaluator state * @param[in] i attribute index - * @param[out] name will store a pointer to the attribute name + * @param[out] name will store a pointer to the attribute name, valid until state is freed * @return value, NULL in case of errors */ nix_value * nix_get_attr_byidx_lazy( nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name); /** @brief Get an attribute name by index + * @ingroup value_extract * * Returns the attribute name without forcing evaluation of the attribute's value. * @@ -382,16 +488,14 @@ nix_value * nix_get_attr_byidx_lazy( * lexicographic order by Unicode scalar value for valid UTF-8). We recommend * applying this same ordering for consistency. * - * Owned by the nix EvalState * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state * @param[in] i attribute index - * @return name, NULL in case of errors + * @return name string valid until state is freed, NULL in case of errors */ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i); -/**@}*/ /** @name Initializers * * Values are typically "returned" by initializing already allocated memory that serves as the return value. @@ -401,6 +505,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, */ /**@{*/ /** @brief Set boolean value + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] b the boolean value @@ -409,6 +514,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); /** @brief Set a string + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] str the string, copied @@ -417,6 +523,7 @@ nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str); /** @brief Set a path + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] str the path string, copied @@ -425,6 +532,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str); /** @brief Set a float + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] d the float, 64-bits @@ -433,6 +541,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); /** @brief Set an int + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] i the int @@ -441,6 +550,7 @@ nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); /** @brief Set null + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @return error code, NIX_OK on success. @@ -448,6 +558,7 @@ nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); nix_err nix_init_null(nix_c_context * context, nix_value * value); /** @brief Set the value to a thunk that will perform a function application when needed. + * @ingroup value_create * * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand. * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp. @@ -464,6 +575,7 @@ nix_err nix_init_null(nix_c_context * context, nix_value * value); nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg); /** @brief Set an external value + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] val the external value to set. Will be GC-referenced by the value. @@ -472,18 +584,25 @@ nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * f nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val); /** @brief Create a list from a list builder + * @ingroup value_create + * + * After this call, the list builder becomes invalid and cannot be used again. + * The only necessary next step is to free it with nix_list_builder_free(). + * * @param[out] context Optional, stores error information - * @param[in] list_builder list builder to use. Make sure to unref this afterwards. + * @param[in] list_builder list builder to use * @param[out] value Nix value to modify * @return error code, NIX_OK on success. + * @see nix_list_builder_free */ nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value); /** @brief Create a list builder + * @ingroup value_create * @param[out] context Optional, stores error information * @param[in] state nix evaluator state * @param[in] capacity how many bindings you'll add. Don't exceed. - * @return owned reference to a list builder. Make sure to unref when you're done. + * @return list builder. Call nix_list_builder_free() when you're done. */ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity); @@ -505,14 +624,21 @@ nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, uns void nix_list_builder_free(ListBuilder * list_builder); /** @brief Create an attribute set from a bindings builder + * @ingroup value_create + * + * After this call, the bindings builder becomes invalid and cannot be used again. + * The only necessary next step is to free it with nix_bindings_builder_free(). + * * @param[out] context Optional, stores error information * @param[out] value Nix value to modify - * @param[in] b bindings builder to use. Make sure to unref this afterwards. + * @param[in] b bindings builder to use * @return error code, NIX_OK on success. + * @see nix_bindings_builder_free */ nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b); /** @brief Set primop + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] op primop, will be gc-referenced by the value @@ -521,6 +647,7 @@ nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuild */ nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op); /** @brief Copy from another value + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] source value to copy from @@ -530,12 +657,11 @@ nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_val /**@}*/ /** @brief Create a bindings builder -* @param[out] context Optional, stores error information -* @param[in] state nix evaluator state -* @param[in] capacity how many bindings you'll add. Don't exceed. -* @return owned reference to a bindings builder. Make sure to unref when you're -done. -*/ + * @param[out] context Optional, stores error information + * @param[in] state nix evaluator state + * @param[in] capacity how many bindings you'll add. Don't exceed. + * @return bindings builder. Call nix_bindings_builder_free() when you're done. + */ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity); /** @brief Insert bindings into a builder @@ -554,7 +680,6 @@ nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, * @param[in] builder the builder to free */ void nix_bindings_builder_free(BindingsBuilder * builder); -/**@}*/ /** @brief Realise a string context. * @@ -571,13 +696,13 @@ void nix_bindings_builder_free(BindingsBuilder * builder); * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false. You should set this to true when this call is part of a primop. You should set this to false when building for your application's purpose. - * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free + * @return NULL if failed, or a new nix_realised_string, which must be freed with nix_realised_string_free */ nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD); /** @brief Start of the string * @param[in] realised_string - * @return pointer to the start of the string. It may not be null-terminated. + * @return pointer to the start of the string, valid until realised_string is freed. It may not be null-terminated. */ const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string); @@ -596,7 +721,7 @@ size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_s /** @brief Get a store path. The store paths are stored in an arbitrary order. * @param[in] realised_string * @param[in] index index of the store path, must be less than the count - * @return store path + * @return store path valid until realised_string is freed */ const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index); @@ -610,5 +735,4 @@ void nix_realised_string_free(nix_realised_string * realised_string); } #endif -/** @} */ #endif // NIX_API_VALUE_H diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 4d7f394fa..d301e5743 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -155,6 +155,8 @@ typedef struct nix_c_context nix_c_context; /** * @brief Called to get the value of a string owned by Nix. * + * The `start` data is borrowed and the function must not assume that the buffer persists after it returns. + * * @param[in] start the string to copy. * @param[in] n the string length. * @param[in] user_data optional, arbitrary data, passed to the nix_get_string_callback when it's called. From 883860c7ff6638f8069d8a6bb1be6ba2065c4608 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Tue, 28 Oct 2025 11:14:31 -0700 Subject: [PATCH 124/201] Move docker documentation to docker.io --- doc/manual/source/installation/installing-docker.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/source/installation/installing-docker.md b/doc/manual/source/installation/installing-docker.md index 9354c1a72..92fa55e1c 100644 --- a/doc/manual/source/installation/installing-docker.md +++ b/doc/manual/source/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti ghcr.io/nixos/nix -Unable to find image 'ghcr.io/nixos/nix:latest' locally -latest: Pulling from ghcr.io/nixos/nix +$ docker run -ti docker.io/nixos/nix +Unable to find image 'docker.io/nixos/nix:latest' locally +latest: Pulling from docker.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for ghcr.io/nixos/nix:latest +Status: Downloaded newer image for docker.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit From 943788754fc695dbe1b8cb3057f7fc1a16858e2c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Tue, 28 Oct 2025 11:16:37 -0700 Subject: [PATCH 125/201] Add ghcr for pre-release --- doc/manual/source/installation/installing-docker.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/source/installation/installing-docker.md b/doc/manual/source/installation/installing-docker.md index 92fa55e1c..ccc75be5a 100644 --- a/doc/manual/source/installation/installing-docker.md +++ b/doc/manual/source/installation/installing-docker.md @@ -16,6 +16,8 @@ nix (Nix) 2.3.12 35ca4ada6e96:/# exit ``` +> If you want the latest pre-release you can use ghcr.io/nixos/nix and view them at https://github.com/nixos/nix/pkgs/container/nix + # What is included in Nix's Docker image? The official Docker image is created using `pkgs.dockerTools.buildLayeredImage` From f5aafbd6ed5ea7a38d27d51cf82d77634d341a05 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 28 Oct 2025 19:39:04 +0100 Subject: [PATCH 126/201] .coderabbit.yaml: Disable auto-review --- .coderabbit.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..5122f01e0 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,6 @@ +# Disable CodeRabbit auto-review to prevent verbose comments on PRs. +# When enabled: false, CodeRabbit won't attempt reviews and won't post +# "Review skipped" or other automated comments. +reviews: + auto_review: + enabled: false From e3246301a6dcd2c722241f4756484d40bc06f48a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 14:49:04 -0400 Subject: [PATCH 127/201] Enable JSON schema testing for derivation outputs I figured out what the problem was: the fragment needs to start with a `/`. --- src/json-schema-checks/meson.build | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 745fb5ffa..8e8ac57c4 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -46,20 +46,19 @@ schemas = [ 'simple-derivation.json', ], }, - # # Not sure how to make subschema work - # { - # 'stem': 'derivation', - # 'schema': schema_dir / 'derivation-v3.yaml#output', - # 'files' : [ - # 'output-caFixedFlat.json', - # 'output-caFixedNAR.json', - # 'output-caFixedText.json', - # 'output-caFloating.json', - # 'output-deferred.json', - # 'output-impure.json', - # 'output-inputAddressed.json', - # ], - # }, + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output', + 'files' : [ + 'output-caFixedFlat.json', + 'output-caFixedNAR.json', + 'output-caFixedText.json', + 'output-caFloating.json', + 'output-deferred.json', + 'output-impure.json', + 'output-inputAddressed.json', + ], + }, { 'stem' : 'deriving-path', 'schema' : schema_dir / 'deriving-path-v1.yaml', From 84a5bee424ab25bd0dbc89b3abc6adb208142396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 28 Oct 2025 21:41:20 +0100 Subject: [PATCH 128/201] coderabbit: disable reporting review status --- .coderabbit.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 5122f01e0..815dc27a5 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -4,3 +4,4 @@ reviews: auto_review: enabled: false + review_status: false From fe8cdbc3e41ecab02d451c8864e6309507d3c7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 28 Oct 2025 21:48:33 +0100 Subject: [PATCH 129/201] coderabbit: disable high_level_summary/poem/github status/sequence_diagrams --- .coderabbit.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 815dc27a5..00244700a 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -5,3 +5,10 @@ reviews: auto_review: enabled: false review_status: false + high_level_summary: false + poem: false + sequence_diagrams: false + changed_files_summary: false + tools: + github-checks: + enabled: false From be2572ed8d0c9dd626462229436ba7aaf2369690 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 17:16:38 -0400 Subject: [PATCH 130/201] Make `inputDrvs` JSON schema more precise It now captures the stable non-recursive format (just an output set) and the unstable recursive form for dynamic derivations. --- .../protocols/json/schema/derivation-v3.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index c950b839f..30fddf699 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -103,6 +103,13 @@ properties: > ``` > > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + additionalProperties: + title: Store Path + description: | + A store path to a derivation, mapped to the outputs of that derivation. + oneOf: + - "$ref": "#/$defs/outputNames" + - "$ref": "#/$defs/dynamicOutputs" system: type: string @@ -167,3 +174,28 @@ properties: title: Expected hash value description: | For fixed-output derivations, the expected content hash in base-16. + + outputName: + type: string + title: Output name + description: Name of the derivation output to depend on + + outputNames: + type: array + title: Output Names + description: Set of names of derivation outputs to depend on + items: + "$ref": "#/$defs/outputName" + + dynamicOutputs: + type: object + title: Dynamic Outputs + description: | + **Experimental feature**: [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) + + This recursive data type allows for depending on outputs of outputs. + properties: + outputs: + "$ref": "#/$defs/outputNames" + dynamicOutputs: + "$ref": "#/$defs/dynamicOutputs" From c67966418f99120a31e3d15c58a0aa253abfb151 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 16:59:35 -0400 Subject: [PATCH 131/201] Create JSON Schema for Store Paths We immediately use this in the JSON schemas for Derivation and Deriving Path, but we cannot yet use it in Store Object Info because those paths *do* include the store dir currently. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/derivation-v3.yaml | 20 ++++++------ .../json/schema/deriving-path-v1.yaml | 2 +- .../protocols/json/schema/store-path-v1 | 1 + .../protocols/json/schema/store-path-v1.yaml | 32 +++++++++++++++++++ .../source/protocols/json/store-path.md | 15 +++++++++ src/json-schema-checks/meson.build | 7 ++++ src/json-schema-checks/package.nix | 1 + src/json-schema-checks/store-path | 1 + 11 files changed, 72 insertions(+), 10 deletions(-) create mode 120000 doc/manual/source/protocols/json/schema/store-path-v1 create mode 100644 doc/manual/source/protocols/json/schema/store-path-v1.yaml create mode 100644 doc/manual/source/protocols/json/store-path.md create mode 120000 src/json-schema-checks/store-path diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 140fa9849..b7c9503ef 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -36,6 +36,7 @@ mkMesonDerivation (finalAttrs: { # For example JSON ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address + ../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/derived-path # Too many different types of files to filter for now ../../doc/manual diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index abd9422cd..7f3b1a103 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -119,6 +119,7 @@ - [JSON Formats](protocols/json/index.md) - [Hash](protocols/json/hash.md) - [Content Address](protocols/json/content-address.md) + - [Store Path](protocols/json/store-path.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index f79667961..e8546d813 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -11,6 +11,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ 'hash-v1', 'content-address-v1', + 'store-path-v1', 'derivation-v3', 'deriving-path-v1', ] diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index 30fddf699..3275bcdd9 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -85,7 +85,7 @@ properties: > ] > ``` items: - type: string + $ref: "store-path-v1.yaml" inputDrvs: type: object @@ -103,13 +103,15 @@ properties: > ``` > > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. - additionalProperties: - title: Store Path - description: | - A store path to a derivation, mapped to the outputs of that derivation. - oneOf: - - "$ref": "#/$defs/outputNames" - - "$ref": "#/$defs/dynamicOutputs" + patternProperties: + "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": + title: Store Path + description: | + A store path to a derivation, mapped to the outputs of that derivation. + oneOf: + - "$ref": "#/$defs/outputNames" + - "$ref": "#/$defs/dynamicOutputs" + additionalProperties: false system: type: string @@ -155,7 +157,7 @@ properties: type: object properties: path: - type: string + $ref: "store-path-v1.yaml" title: Output path description: | The output path, if known in advance. diff --git a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml index 7fd74941e..11a784d06 100644 --- a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml @@ -7,7 +7,7 @@ oneOf: - title: Constant description: | See [Constant](@docroot@/store/derivation/index.md#deriving-path-constant) deriving path. - type: string + $ref: "store-path-v1.yaml" - title: Output description: | See [Output](@docroot@/store/derivation/index.md#deriving-path-output) deriving path. diff --git a/doc/manual/source/protocols/json/schema/store-path-v1 b/doc/manual/source/protocols/json/schema/store-path-v1 new file mode 120000 index 000000000..31e7a6b2a --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-path-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/store-path \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml new file mode 100644 index 000000000..2012aab99 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -0,0 +1,32 @@ +"$schema": "http://json-schema.org/draft-07/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-path-v1.json" +title: Store Path +description: | + A [store path](@docroot@/store/store-path.md) identifying a store object. + + This schema describes the JSON representation of store paths as used in various Nix JSON APIs. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + + ## Format + + Store paths in JSON are represented as strings containing just the hash and name portion, without the store directory prefix. + + For example: `"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"` + + (If the store dir is `/nix/store`, then this corresponds to the path `/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv`.) + + ## Structure + + The format follows this pattern: `${digest}-${name}` + + - **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - **name**: The package name and optional version/suffix information + +type: string +pattern: "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$" +minLength: 34 diff --git a/doc/manual/source/protocols/json/store-path.md b/doc/manual/source/protocols/json/store-path.md new file mode 100644 index 000000000..02ecc8068 --- /dev/null +++ b/doc/manual/source/protocols/json/store-path.md @@ -0,0 +1,15 @@ +{{#include store-path-v1-fixed.md}} + +## Examples + +### Simple store path + +```json +{{#include schema/store-path-v1/simple.json}} +``` + + diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 745fb5ffa..f3e52e544 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -38,6 +38,13 @@ schemas = [ 'nar.json', ], }, + { + 'stem' : 'store-path', + 'schema' : schema_dir / 'store-path-v1.yaml', + 'files' : [ + 'simple.json', + ], + }, { 'stem' : 'derivation', 'schema' : schema_dir / 'derivation-v3.yaml', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 6a76c8b28..0122b5493 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -22,6 +22,7 @@ mkMesonDerivation (finalAttrs: { ../../doc/manual/source/protocols/json/schema ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address + ../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/derivation ../../src/libstore-tests/data/derived-path ./. diff --git a/src/json-schema-checks/store-path b/src/json-schema-checks/store-path new file mode 120000 index 000000000..003b1dbbb --- /dev/null +++ b/src/json-schema-checks/store-path @@ -0,0 +1 @@ +../../src/libstore-tests/data/store-path \ No newline at end of file From c874e7071b0f81406a4078e5ce0aec50770ccd53 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 29 Oct 2025 01:47:18 +0300 Subject: [PATCH 132/201] libstore/http-binary-cache-store: Improve error messages in HttpBinaryCacheStore::upsertFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now the error message doesn't cram everything into a single line and we now instead get: error: … while uploading to HTTP binary cache at 's3://my-cache?endpoint=http://localhost:9000?compression%3Dzstd®ion=eu-west-1' error: unable to download 'http://localhost:9000/my-cache/nar/1125zqba8cx8wbfa632vy458a3j3xja0qpcqafsfdildyl9dqa7x.nar.xz': Operation was aborted by an application callback (42) --- src/libstore/http-binary-cache-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 738db132d..1f9ee4100 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -157,7 +157,9 @@ void HttpBinaryCacheStore::upsertFile( try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg()); + UploadToHTTP err(e.message()); + err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); + throw err; } } From ae49074548bb3485a0a263ca862f6aee95cfb09f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 29 Oct 2025 02:48:26 +0300 Subject: [PATCH 133/201] libstore/filetransfer: Add HttpMethod::PUT This got lost in f1968ea38e51201b37962a9cfd80775989a56d46 and now we had incorrect logs that confused "downloading" when we were in fact "uploading" things. --- src/libstore/filetransfer.cc | 4 +++- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/include/nix/store/filetransfer.hh | 3 +++ src/libstore/s3-binary-cache-store.cc | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 6b9c6602b..304984d99 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -394,9 +394,11 @@ struct curlFileTransfer : public FileTransfer if (request.method == HttpMethod::POST) { curl_easy_setopt(req, CURLOPT_POST, 1L); curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->length()); - } else { + } else if (request.method == HttpMethod::PUT) { curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + } else { + unreachable(); } curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); curl_easy_setopt(req, CURLOPT_READDATA, this); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 738db132d..089c7873a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -141,7 +141,7 @@ void HttpBinaryCacheStore::upsertFile( uint64_t sizeHint) { auto req = makeRequest(path); - + req.method = HttpMethod::PUT; auto data = StreamToSourceAdapter(istream).drain(); auto compressionMethod = getCompressionMethod(path); diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 305c33af1..08a2b6329 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -88,6 +88,7 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT; */ enum struct HttpMethod { GET, + PUT, HEAD, POST, DELETE, @@ -147,7 +148,9 @@ struct FileTransferRequest case HttpMethod::HEAD: case HttpMethod::GET: return "download"; + case HttpMethod::PUT: case HttpMethod::POST: + assert(data); return "upload"; case HttpMethod::DELETE: return "delet"; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 828e75b7c..417355b68 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -101,6 +101,7 @@ std::string S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data) { auto req = makeRequest(key); + req.method = HttpMethod::PUT; req.setupForS3(); auto url = req.uri.parsed(); From 6280905638aac9d15c09fc4d38aa469ee63d17be Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 13:24:05 -0400 Subject: [PATCH 134/201] Convert store path info JSON docs to formal JSON Schema, and test This continues the work for formalizing our current JSON docs. Note that in the process, a few bugs were caught: - `closureSize` was repeated twice, forgot `closureDownloadSize` - `file*` fields should be `download*`. They are in fact called that in the line-oriented `.narinfo` file, but were renamed in the JSON format. --- doc/manual/package.nix | 2 + .../source/protocols/json/derivation.md | 2 +- doc/manual/source/protocols/json/hash.md | 2 +- doc/manual/source/protocols/json/meson.build | 1 + .../source/protocols/json/schema/nar-info-v1 | 1 + .../json/schema/store-object-info-v1 | 1 + .../json/schema/store-object-info-v1.yaml | 235 ++++++++++++++++++ .../protocols/json/store-object-info.md | 117 +++------ .../source/protocols/json/store-path.md | 2 +- src/json-schema-checks/meson.build | 50 ++++ src/json-schema-checks/nar-info | 1 + src/json-schema-checks/package.nix | 2 + src/json-schema-checks/store-object-info | 1 + 13 files changed, 327 insertions(+), 90 deletions(-) create mode 120000 doc/manual/source/protocols/json/schema/nar-info-v1 create mode 120000 doc/manual/source/protocols/json/schema/store-object-info-v1 create mode 100644 doc/manual/source/protocols/json/schema/store-object-info-v1.yaml create mode 120000 src/json-schema-checks/nar-info create mode 120000 src/json-schema-checks/store-object-info diff --git a/doc/manual/package.nix b/doc/manual/package.nix index b7c9503ef..7d29df3c3 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -38,6 +38,8 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/derived-path + ../../src/libstore-tests/data/path-info + ../../src/libstore-tests/data/nar-info # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 602ab67e4..a4a4ea79d 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,6 +1,6 @@ {{#include derivation-v3-fixed.md}} - diff --git a/doc/manual/source/protocols/json/store-path.md b/doc/manual/source/protocols/json/store-path.md index 02ecc8068..cd18f6595 100644 --- a/doc/manual/source/protocols/json/store-path.md +++ b/doc/manual/source/protocols/json/store-path.md @@ -8,7 +8,7 @@ {{#include schema/store-path-v1/simple.json}} ``` - For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it passes the `-j${NIX_BUILD_CORES}` flag to GNU Make. - If set to `0`, nix will detect the number of CPU cores and pass this number via NIX_BUILD_CORES. + If set to `0`, nix will detect the number of CPU cores and pass this number via `NIX_BUILD_CORES`. > **Note** > diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index cb105a385..33ad8757a 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -87,7 +87,7 @@ struct AuthorizationSettings : Config {"*"}, "allowed-users", R"( - A list user names, separated by whitespace. + A list of user names, separated by whitespace. These users are allowed to connect to the Nix daemon. You can specify groups by prefixing names with `@`. From 4ea32d0b03f04143c54344363affea50fc804681 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 2 Nov 2025 14:00:07 +0100 Subject: [PATCH 166/201] Improve "resolution failed" error Previously: error: Cannot build '/nix/store/cqc798lwy2njwbdzgd0319z4r19j2d1w-nix-manual-2.33.0pre20251101_e4e4063.drv'. Reason: 1 dependency failed. Output paths: /nix/store/f1kln1c6z9r7rlhj0h9shcpch7j5g1fj-nix-manual-2.33.0pre20251101_e4e4063-man /nix/store/k65203rx5g1kcagpcz3c3a09bghcj92a-nix-manual-2.33.0pre20251101_e4e4063 error: Cannot build '/nix/store/ajk2fb6r7ijn2fc5c3h85n6zdi36xlfl-nixops-manual.drv'. Reason: 1 dependency failed. Output paths: /nix/store/0anr0998as8ry4hr5g3f3iarszx5aisx-nixops-manual error: resolution failed Now: error: Cannot build '/nix/store/cqc798lwy2njwbdzgd0319z4r19j2d1w-nix-manual-2.33.0pre20251101_e4e4063.drv'. Reason: 1 dependency failed. Output paths: /nix/store/f1kln1c6z9r7rlhj0h9shcpch7j5g1fj-nix-manual-2.33.0pre20251101_e4e4063-man /nix/store/k65203rx5g1kcagpcz3c3a09bghcj92a-nix-manual-2.33.0pre20251101_e4e4063 error: Cannot build '/nix/store/ajk2fb6r7ijn2fc5c3h85n6zdi36xlfl-nixops-manual.drv'. Reason: 1 dependency failed. Output paths: /nix/store/0anr0998as8ry4hr5g3f3iarszx5aisx-nixops-manual error: Build failed due to failed dependency --- src/libstore/build/derivation-goal.cc | 2 +- tests/functional/build.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 717d6890a..14aa044ea 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -147,7 +147,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) co_await await(std::move(waitees)); } if (nrFailed != 0) { - co_return doneFailure({BuildResult::Failure::DependencyFailed, "resolution failed"}); + co_return doneFailure({BuildResult::Failure::DependencyFailed, "Build failed due to failed dependency"}); } if (resolutionGoal->resolvedDrv) { diff --git a/tests/functional/build.sh b/tests/functional/build.sh index c9a39438d..0b06dcd91 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -184,6 +184,7 @@ test "$status" = 1 if isDaemonNewer "2.29pre"; then <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" <<<"$out" grepQuiet -E "Reason: 1 dependency failed." + <<<"$out" grepQuiet -E "Build failed due to failed dependency" else <<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" fi From 233bd250d175719896ef4985acb4a41613cb34c9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 2 Nov 2025 14:10:12 +0100 Subject: [PATCH 167/201] flake: Update, nixos-25.05-small -> nixos-25.05 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d98ce345cdab58477ca61855540999c86577d19d?narHash=sha256-O2CIn7HjZwEGqBrwu9EU76zlmA5dbmna7jL1XUmAId8%3D' (2025-08-26) → 'github:NixOS/nixpkgs/daf6dc47aa4b44791372d6139ab7b25269184d55?narHash=sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8%2BON/0Yy8%2Ba5vsDU%3D' (2025-10-27) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index cc2b2f27e..63290ef86 100644 --- a/flake.lock +++ b/flake.lock @@ -63,16 +63,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1756178832, - "narHash": "sha256-O2CIn7HjZwEGqBrwu9EU76zlmA5dbmna7jL1XUmAId8=", + "lastModified": 1761597516, + "narHash": "sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8+ON/0Yy8+a5vsDU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d98ce345cdab58477ca61855540999c86577d19d", + "rev": "daf6dc47aa4b44791372d6139ab7b25269184d55", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-25.05-small", + "ref": "nixos-25.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 418f3180f..e25722d46 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; From bf947bfc26704b3a21da222f3c67fb9d773383b9 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Tue, 21 Oct 2025 06:19:17 +0000 Subject: [PATCH 168/201] feat(libstore/s3-binary-cache-store): add multipart upload config settings Add three configuration settings to `S3BinaryCacheStoreConfig` to control multipart upload behavior: - `bool multipart-upload` (default `false`): Enable/disable multipart uploads - `uint64_t multipart-chunk-size` (default 5 MiB): Size of each upload part - `uint64_t multipart-threshold` (default 100 MiB): Minimum file size for multipart The feature is disabled by default. --- .../nix/store/s3-binary-cache-store.hh | 32 +++++++++++++++++++ src/libstore/s3-binary-cache-store.cc | 23 +++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 81a2d3f3f..bf86d0671 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -61,6 +61,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig > addressing instead of virtual host based addressing. )"}; + const Setting multipartUpload{ + this, + false, + "multipart-upload", + R"( + Whether to use multipart uploads for large files. When enabled, + files exceeding the multipart threshold will be uploaded in + multiple parts, which is required for files larger than 5 GiB and + can improve performance and reliability for large uploads. + )"}; + + const Setting multipartChunkSize{ + this, + 5 * 1024 * 1024, + "multipart-chunk-size", + R"( + The size (in bytes) of each part in multipart uploads. Must be + at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the + number of requests but use more memory. Default is 5 MiB. + )", + {"buffer-size"}}; + + const Setting multipartThreshold{ + this, + 100 * 1024 * 1024, + "multipart-threshold", + R"( + The minimum file size (in bytes) for using multipart uploads. + Files smaller than this threshold will use regular PUT requests. + Default is 100 MiB. Only takes effect when multipart-upload is enabled. + )"}; + /** * Set of settings that are part of the S3 URI itself. * These are needed for region specification and other S3-specific settings. diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 9303a80f8..4cf5f987a 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -15,6 +15,7 @@ namespace nix { MakeError(UploadToS3, Error); +static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB class S3BinaryCacheStore : public virtual HttpBinaryCacheStore @@ -253,6 +254,28 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( cacheUri.query[key] = value; } } + + if (multipartChunkSize < AWS_MIN_PART_SIZE) { + throw UsageError( + "multipart-chunk-size must be at least %s, got %s", + renderSize(AWS_MIN_PART_SIZE), + renderSize(multipartChunkSize.get())); + } + + if (multipartChunkSize > AWS_MAX_PART_SIZE) { + throw UsageError( + "multipart-chunk-size must be at most %s, got %s", + renderSize(AWS_MAX_PART_SIZE), + renderSize(multipartChunkSize.get())); + } + + if (multipartUpload && multipartThreshold < multipartChunkSize) { + warn( + "multipart-threshold (%s) is less than multipart-chunk-size (%s), " + "which may result in single-part multipart uploads", + renderSize(multipartThreshold.get()), + renderSize(multipartChunkSize.get())); + } } std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const From 040d1aae41a3bfda86c29910eb1495d75598fd35 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 08:42:32 +0000 Subject: [PATCH 169/201] feat(libstore/s3-binary-cache-store): implement `uploadMultipart()` Implement `uploadMultipart()`, the main method that orchestrates S3 multipart uploads --- src/libstore/s3-binary-cache-store.cc | 228 ++++++++++++++++++++++---- 1 file changed, 192 insertions(+), 36 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4cf5f987a..37264dfae 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,7 @@ #include "nix/util/util.hh" #include +#include #include #include #include @@ -17,6 +18,7 @@ MakeError(UploadToS3, Error); static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB +static constexpr uint64_t AWS_MAX_PART_COUNT = 10000; class S3BinaryCacheStore : public virtual HttpBinaryCacheStore { @@ -51,9 +53,48 @@ private: std::optional contentEncoding); /** - * Uploads a file to S3 (CompressedSource overload). + * Uploads a file to S3 using multipart upload. + * + * This method is suitable for large files that exceed the multipart threshold. + * It orchestrates the complete multipart upload process: creating the upload, + * splitting the data into parts, uploading each part, and completing the upload. + * If any error occurs, the multipart upload is automatically aborted. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html */ - void upload(std::string_view path, CompressedSource & source, std::string_view mimeType); + void uploadMultipart( + std::string_view path, + RestartableSource & source, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding); + + /** + * A Sink that manages a complete S3 multipart upload lifecycle. + * Creates the upload on construction, buffers and uploads chunks as data arrives, + * and completes or aborts the upload appropriately. + */ + struct MultipartSink : Sink + { + S3BinaryCacheStore & store; + std::string_view path; + std::string uploadId; + std::string::size_type chunkSize; + + std::vector partEtags; + std::string buffer; + + MultipartSink( + S3BinaryCacheStore & store, + std::string_view path, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding); + + void operator()(std::string_view data) override; + void finish(); + void uploadChunk(std::string chunk); + }; /** * Creates a multipart upload for large objects to S3. @@ -73,18 +114,13 @@ private: */ std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data); - struct UploadedPart - { - uint64_t partNumber; - std::string etag; - }; - /** * Completes a multipart upload by combining all uploaded parts. * @see * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_RequestSyntax */ - void completeMultipartUpload(std::string_view key, std::string_view uploadId, std::span parts); + void + completeMultipartUpload(std::string_view key, std::string_view uploadId, std::span partEtags); /** * Abort a multipart upload @@ -92,17 +128,31 @@ private: * @see * https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html#API_AbortMultipartUpload_RequestSyntax */ - void abortMultipartUpload(std::string_view key, std::string_view uploadId); + void abortMultipartUpload(std::string_view key, std::string_view uploadId) noexcept; }; void S3BinaryCacheStore::upsertFile( const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) { - if (auto compressionMethod = getCompressionMethod(path)) { - CompressedSource compressed(source, *compressionMethod); - upload(path, compressed, mimeType); - } else { - upload(path, source, sizeHint, mimeType, std::nullopt); + auto doUpload = [&](RestartableSource & src, uint64_t size, std::optional encoding) { + if (s3Config->multipartUpload && size > s3Config->multipartThreshold) { + uploadMultipart(path, src, size, mimeType, encoding); + } else { + upload(path, src, size, mimeType, encoding); + } + }; + + try { + if (auto compressionMethod = getCompressionMethod(path)) { + CompressedSource compressed(source, *compressionMethod); + doUpload(compressed, compressed.size(), compressed.getCompressionMethod()); + } else { + doUpload(source, sizeHint, std::nullopt); + } + } catch (FileTransferError & e) { + UploadToS3 err(e.message()); + err.addTrace({}, "while uploading to S3 binary cache at '%s'", config->cacheUri.to_string()); + throw err; } } @@ -120,18 +170,112 @@ void S3BinaryCacheStore::upload( renderSize(sizeHint), renderSize(AWS_MAX_PART_SIZE)); - try { - HttpBinaryCacheStore::upload(path, source, sizeHint, mimeType, contentEncoding); - } catch (FileTransferError & e) { - UploadToS3 err(e.message()); - err.addTrace({}, "while uploading to S3 binary cache at '%s'", config->cacheUri.to_string()); - throw err; + HttpBinaryCacheStore::upload(path, source, sizeHint, mimeType, contentEncoding); +} + +void S3BinaryCacheStore::uploadMultipart( + std::string_view path, + RestartableSource & source, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding) +{ + debug("using S3 multipart upload for '%s' (%d bytes)", path, sizeHint); + MultipartSink sink(*this, path, sizeHint, mimeType, contentEncoding); + source.drainInto(sink); + sink.finish(); +} + +S3BinaryCacheStore::MultipartSink::MultipartSink( + S3BinaryCacheStore & store, + std::string_view path, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding) + : store(store) + , path(path) +{ + // Calculate chunk size and estimated parts + chunkSize = store.s3Config->multipartChunkSize; + uint64_t estimatedParts = (sizeHint + chunkSize - 1) / chunkSize; // ceil division + + if (estimatedParts > AWS_MAX_PART_COUNT) { + // Equivalent to ceil(sizeHint / AWS_MAX_PART_COUNT) + uint64_t minChunkSize = (sizeHint + AWS_MAX_PART_COUNT - 1) / AWS_MAX_PART_COUNT; + + if (minChunkSize > AWS_MAX_PART_SIZE) { + throw Error( + "file too large for S3 multipart upload: %s would require chunk size of %s " + "(max %s) to stay within %d part limit", + renderSize(sizeHint), + renderSize(minChunkSize), + renderSize(AWS_MAX_PART_SIZE), + AWS_MAX_PART_COUNT); + } + + warn( + "adjusting S3 multipart chunk size from %s to %s " + "to stay within %d part limit for %s file", + renderSize(store.s3Config->multipartChunkSize.get()), + renderSize(minChunkSize), + AWS_MAX_PART_COUNT, + renderSize(sizeHint)); + + chunkSize = minChunkSize; + estimatedParts = AWS_MAX_PART_COUNT; + } + + buffer.reserve(chunkSize); + partEtags.reserve(estimatedParts); + uploadId = store.createMultipartUpload(path, mimeType, contentEncoding); +} + +void S3BinaryCacheStore::MultipartSink::operator()(std::string_view data) +{ + buffer.append(data); + + while (buffer.size() >= chunkSize) { + // Move entire buffer, extract excess, copy back remainder + auto chunk = std::move(buffer); + auto excessSize = chunk.size() > chunkSize ? chunk.size() - chunkSize : 0; + if (excessSize > 0) { + buffer.resize(excessSize); + std::memcpy(buffer.data(), chunk.data() + chunkSize, excessSize); + } + chunk.resize(std::min(chunkSize, chunk.size())); + uploadChunk(std::move(chunk)); } } -void S3BinaryCacheStore::upload(std::string_view path, CompressedSource & source, std::string_view mimeType) +void S3BinaryCacheStore::MultipartSink::finish() { - upload(path, static_cast(source), source.size(), mimeType, source.getCompressionMethod()); + if (!buffer.empty()) { + uploadChunk(std::move(buffer)); + } + + try { + if (partEtags.empty()) { + throw Error("no data read from stream"); + } + store.completeMultipartUpload(path, uploadId, partEtags); + } catch (Error & e) { + store.abortMultipartUpload(path, uploadId); + e.addTrace({}, "while finishing an S3 multipart upload"); + throw; + } +} + +void S3BinaryCacheStore::MultipartSink::uploadChunk(std::string chunk) +{ + auto partNumber = partEtags.size() + 1; + try { + std::string etag = store.uploadPart(path, uploadId, partNumber, std::move(chunk)); + partEtags.push_back(std::move(etag)); + } catch (Error & e) { + store.abortMultipartUpload(path, uploadId); + e.addTrace({}, "while uploading part %d of an S3 multipart upload", partNumber); + throw; + } } std::string S3BinaryCacheStore::createMultipartUpload( @@ -171,6 +315,10 @@ std::string S3BinaryCacheStore::createMultipartUpload( std::string S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data) { + if (partNumber > AWS_MAX_PART_COUNT) { + throw Error("S3 multipart upload exceeded %d part limit", AWS_MAX_PART_COUNT); + } + auto req = makeRequest(key); req.method = HttpMethod::PUT; req.setupForS3(); @@ -189,24 +337,29 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, throw Error("S3 UploadPart response missing ETag for part %d", partNumber); } + debug("Part %d uploaded, ETag: %s", partNumber, result.etag); return std::move(result.etag); } -void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) +void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) noexcept { - auto req = makeRequest(key); - req.setupForS3(); + try { + auto req = makeRequest(key); + req.setupForS3(); - auto url = req.uri.parsed(); - url.query["uploadId"] = uploadId; - req.uri = VerbatimURL(url); - req.method = HttpMethod::DELETE; + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::DELETE; - getFileTransfer()->enqueueFileTransfer(req).get(); + getFileTransfer()->enqueueFileTransfer(req).get(); + } catch (...) { + ignoreExceptionInDestructor(); + } } void S3BinaryCacheStore::completeMultipartUpload( - std::string_view key, std::string_view uploadId, std::span parts) + std::string_view key, std::string_view uploadId, std::span partEtags) { auto req = makeRequest(key); req.setupForS3(); @@ -217,21 +370,24 @@ void S3BinaryCacheStore::completeMultipartUpload( req.method = HttpMethod::POST; std::string xml = ""; - for (const auto & part : parts) { + for (const auto & [idx, etag] : enumerate(partEtags)) { xml += ""; - xml += "" + std::to_string(part.partNumber) + ""; - xml += "" + part.etag + ""; + // S3 part numbers are 1-indexed, but vector indices are 0-indexed + xml += "" + std::to_string(idx + 1) + ""; + xml += "" + etag + ""; xml += ""; } xml += ""; - debug("S3 CompleteMultipartUpload XML (%d parts): %s", parts.size(), xml); + debug("S3 CompleteMultipartUpload XML (%d parts): %s", partEtags.size(), xml); StringSource payload{xml}; req.data = {payload}; req.mimeType = "text/xml"; getFileTransfer()->enqueueFileTransfer(req).get(); + + debug("S3 multipart upload completed: %d parts uploaded for '%s'", partEtags.size(), key); } StringSet S3BinaryCacheStoreConfig::uriSchemes() From 965d6be7c1962b87d47eb229153e2b5685c64739 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Thu, 30 Oct 2025 19:22:06 +0000 Subject: [PATCH 170/201] tests(nixos/s3-binary-cache-store): enable multipart --- tests/nixos/s3-binary-cache-store.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index a07375489..a2ba1dae6 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -794,10 +794,9 @@ in test_compression_disabled() test_nix_prefetch_url() test_versioned_urls() - # FIXME: enable when multipart fully lands - # test_multipart_upload_basic() - # test_multipart_threshold() - # test_multipart_with_log_compression() + test_multipart_upload_basic() + test_multipart_threshold() + test_multipart_with_log_compression() print("\n" + "="*80) print("✓ All S3 Binary Cache Store Tests Passed!") From 3448d4fa4c7fca0d62487fa6ac0dfded72ff18de Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 05:05:10 +0000 Subject: [PATCH 171/201] docs(rl-next/s3-curl-implementation): update with multipart uploads --- doc/manual/rl-next/s3-curl-implementation.md | 24 ++++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/doc/manual/rl-next/s3-curl-implementation.md b/doc/manual/rl-next/s3-curl-implementation.md index fab010010..2647ac581 100644 --- a/doc/manual/rl-next/s3-curl-implementation.md +++ b/doc/manual/rl-next/s3-curl-implementation.md @@ -1,6 +1,6 @@ --- synopsis: "Improved S3 binary cache support via HTTP" -prs: [13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 13752] +prs: [13752, 13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 14330, 14333, 14335, 14336, 14337, 14350, 14356, 14357, 14374, 14375, 14376, 14377, 14391, 14393, 14420, 14421] issues: [13084, 12671, 11748, 12403] --- @@ -18,9 +18,23 @@ improvements: The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential management. -All existing S3 URL formats and parameters remain supported, with the notable -exception of multi-part uploads, which are no longer supported. +All existing S3 URL formats and parameters remain supported, however the store +settings for configuring multipart uploads have changed: + +- **`multipart-upload`** (default: `false`): Enable multipart uploads for large + files. When enabled, files exceeding the multipart threshold will be uploaded + in multiple parts. + +- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using + multipart uploads. Files smaller than this will use regular PUT requests. + Only takes effect when `multipart-upload` is enabled. + +- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart + uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes + reduce the number of requests but use more memory. + +- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it. Note that this change also means Nix now supports S3 binary cache stores even -if build without `aws-crt-cpp`, but only for public buckets which do not -require auth. +if built without `aws-crt-cpp`, but only for public buckets which do not +require authentication. From 81a2809a526e4fcc887d3178c8b48646320a25e8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Nov 2025 12:01:55 +0100 Subject: [PATCH 172/201] Apply updated nixfmt --- doc/manual/generate-store-types.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix index a03d3d621..4e06c7f60 100644 --- a/doc/manual/generate-store-types.nix +++ b/doc/manual/generate-store-types.nix @@ -24,9 +24,9 @@ let in concatStringsSep "\n" (map showEntry storesList); - "index.md" = - replaceStrings [ "@store-types@" ] [ index ] - (readFile ./source/store/types/index.md.in); + "index.md" = replaceStrings [ "@store-types@" ] [ index ] ( + readFile ./source/store/types/index.md.in + ); tableOfContents = let From bd420928730cf268ffe33071292e044118a0c57c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Nov 2025 16:54:22 -0400 Subject: [PATCH 173/201] Use less `c_str()` in the evaluator, and other cleanups It is better to avoid null termination for performance and memory safety, wherever possible. These are good cleanups extracted from the Pascal String work that we can land by themselves first, shrinking the diff in that PR. Co-Authored-By: Aspen Smith Co-Authored-By: Sergei Zimmerman --- src/libexpr-c/nix_api_value.cc | 2 +- .../include/nix/expr/tests/libexpr.hh | 2 +- src/libexpr-tests/value/value.cc | 18 +++++++++++++++ src/libexpr/eval-cache.cc | 6 ++--- src/libexpr/eval.cc | 23 +++++++++++-------- src/libexpr/get-drvs.cc | 8 +++---- src/libexpr/include/nix/expr/get-drvs.hh | 2 +- src/libexpr/include/nix/expr/value.hh | 7 +++++- src/libexpr/nixexpr.cc | 2 +- src/libexpr/primops.cc | 6 ++--- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 6 ++--- src/libflake/flake.cc | 6 ++--- src/libutil-c/nix_api_util.cc | 4 ++-- src/libutil-c/nix_api_util_internal.h | 2 +- src/nix/nix-env/nix-env.cc | 6 ++--- 16 files changed, 64 insertions(+), 38 deletions(-) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 3b8c7dd04..e231c36f4 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -235,7 +235,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_ try { auto & v = check_value_in(value); assert(v.type() == nix::nString); - call_nix_get_string_callback(v.c_str(), callback, user_data); + call_nix_get_string_callback(v.string_view(), callback, user_data); } NIXC_CATCH_ERRS } diff --git a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh index a1320e14a..daae00802 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh @@ -106,7 +106,7 @@ MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s)) if (arg.type() != nString) { return false; } - return std::string_view(arg.c_str()) == s; + return arg.string_view() == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) diff --git a/src/libexpr-tests/value/value.cc b/src/libexpr-tests/value/value.cc index 63501dd49..229e449db 100644 --- a/src/libexpr-tests/value/value.cc +++ b/src/libexpr-tests/value/value.cc @@ -1,6 +1,7 @@ #include "nix/expr/value.hh" #include "nix/store/tests/libstore.hh" +#include namespace nix { @@ -22,4 +23,21 @@ TEST_F(ValueTest, vInt) ASSERT_EQ(true, vInt.isValid()); } +TEST_F(ValueTest, staticString) +{ + Value vStr1; + Value vStr2; + vStr1.mkStringNoCopy("foo"); + vStr2.mkStringNoCopy("foo"); + + auto sd1 = vStr1.string_view(); + auto sd2 = vStr2.string_view(); + + // The strings should be the same + ASSERT_EQ(sd1, sd2); + + // The strings should also be backed by the same (static) allocation + ASSERT_EQ(sd1.data(), sd2.data()); +} + } // namespace nix diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 480ca72c7..de74d2143 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -406,7 +406,7 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), string_t{v.c_str(), {}}}; + cachedValue = {root->db->setString(getKey(), v.string_view(), v.context()), string_t{v.string_view(), {}}}; else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; @@ -541,7 +541,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow(); - return v.type() == nString ? v.c_str() : v.path().to_string(); + return v.type() == nString ? std::string(v.string_view()) : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -580,7 +580,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) { NixStringContext context; copyContext(v, context); - return {v.c_str(), std::move(context)}; + return {std::string{v.string_view()}, std::move(context)}; } else if (v.type() == nPath) return {v.path().to_string(), {}}; else diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 873b88986..e2687148b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2366,12 +2366,15 @@ BackedStringView EvalState::coerceToString( } if (v.type() == nPath) { - return !canonicalizePath && !copyToStore - ? // FIXME: hack to preserve path literals that end in a - // slash, as in /foo/${x}. - v.pathStr() - : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) - : std::string(v.path().path.abs()); + if (!canonicalizePath && !copyToStore) { + // FIXME: hack to preserve path literals that end in a + // slash, as in /foo/${x}. + return v.pathStrView(); + } else if (copyToStore) { + return store->printStorePath(copyPathToStore(context, v.path())); + } else { + return std::string{v.path().path.abs()}; + } } if (v.type() == nAttrs) { @@ -2624,7 +2627,7 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st return; case nString: - if (strcmp(v1.c_str(), v2.c_str()) != 0) { + if (v1.string_view() != v2.string_view()) { error( "string '%s' is not equal to string '%s'", ValuePrinter(*this, v1, errorPrintOptions), @@ -2641,7 +2644,7 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st ValuePrinter(*this, v2, errorPrintOptions)) .debugThrow(); } - if (strcmp(v1.pathStr(), v2.pathStr()) != 0) { + if (v1.pathStrView() != v2.pathStrView()) { error( "path '%s' is not equal to path '%s'", ValuePrinter(*this, v1, errorPrintOptions), @@ -2807,12 +2810,12 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean() == v2.boolean(); case nString: - return strcmp(v1.c_str(), v2.c_str()) == 0; + return v1.string_view() == v2.string_view(); case nPath: return // FIXME: compare accessors by their fingerprint. - v1.pathAccessor() == v2.pathAccessor() && strcmp(v1.pathStr(), v2.pathStr()) == 0; + v1.pathAccessor() == v2.pathAccessor() && v1.pathStrView() == v2.pathStrView(); case nNull: return true; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 5a7281b2b..c4a2b00af 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -168,7 +168,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT for (auto elem : outTI->listView()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->c_str()); + auto out = outputs.find(elem->string_view()); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -245,7 +245,7 @@ std::string PackageInfo::queryMetaString(const std::string & name) Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->c_str(); + return std::string{v->string_view()}; } NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) @@ -258,7 +258,7 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->c_str())) + if (auto n = string2Int(v->string_view())) return NixInt{*n}; } return def; @@ -274,7 +274,7 @@ NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->c_str())) + if (auto n = string2Float(v->string_view())) return *n; } return def; diff --git a/src/libexpr/include/nix/expr/get-drvs.hh b/src/libexpr/include/nix/expr/get-drvs.hh index 3d42188bf..4beccabe2 100644 --- a/src/libexpr/include/nix/expr/get-drvs.hh +++ b/src/libexpr/include/nix/expr/get-drvs.hh @@ -15,7 +15,7 @@ namespace nix { struct PackageInfo { public: - typedef std::map> Outputs; + typedef std::map, std::less<>> Outputs; private: EvalState * state; diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 22d85dc99..706a4fe3f 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -1109,7 +1109,7 @@ public: std::string_view string_view() const noexcept { - return std::string_view(getStorage().c_str); + return std::string_view{getStorage().c_str}; } const char * c_str() const noexcept @@ -1177,6 +1177,11 @@ public: return getStorage().path; } + std::string_view pathStrView() const noexcept + { + return std::string_view{getStorage().path}; + } + SourceAccessor * pathAccessor() const noexcept { return getStorage().accessor; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index b183f1bbf..a1d1b7e4b 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -45,7 +45,7 @@ void ExprString::show(const SymbolTable & symbols, std::ostream & str) const void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const { - str << v.pathStr(); + str << v.pathStrView(); } void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 04196bc1f..96e79fedd 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -691,12 +691,12 @@ struct CompareValues case nFloat: return v1->fpoint() < v2->fpoint(); case nString: - return strcmp(v1->c_str(), v2->c_str()) < 0; + return v1->string_view() < v2->string_view(); case nPath: // Note: we don't take the accessor into account // since it's not obvious how to compare them in a // reproducible way. - return strcmp(v1->pathStr(), v2->pathStr()) < 0; + return v1->pathStrView() < v2->pathStrView(); case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -2930,7 +2930,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value ** args, V for (const auto & [n, i] : enumerate(*args[0]->attrs())) list[n] = Value::toPtr(state.symbols[i.name]); - std::sort(list.begin(), list.end(), [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; }); + std::sort(list.begin(), list.end(), [](Value * v1, Value * v2) { return v1->string_view() < v2->string_view(); }); v.mkList(list); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 2cd853f60..03b14b83c 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -33,7 +33,7 @@ json printValueAsJSON( case nString: copyContext(v, context); - out = v.c_str(); + out = v.string_view(); break; case nPath: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index d5959e894..0a7a334f4 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -82,7 +82,7 @@ static void printValueAsXML( case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); + doc.writeEmptyElement("string", singletonAttrs("value", v.string_view())); break; case nPath: @@ -102,14 +102,14 @@ static void printValueAsXML( if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->c_str(); + xmlAttrs["drvPath"] = drvPath = a->value->string_view(); } if (auto a = v.attrs()->get(state.s.outPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->c_str(); + xmlAttrs["outPath"] = a->value->string_view(); } XMLOpenElement _(doc, "derivation", xmlAttrs); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 42385712c..dc60dbf08 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -97,7 +97,7 @@ static void parseFlakeInputAttr(EvalState & state, const Attr & attr, fetchers:: #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->c_str()); + attrs.emplace(state.symbols[attr.name], std::string(attr.value->string_view())); break; case nBool: attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); @@ -177,7 +177,7 @@ static FlakeInput parseFlakeInput( parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputAttrPath(attr.value->c_str())); + auto follows(parseInputAttrPath(attr.value->string_view())); follows.insert(follows.begin(), lockRootAttrPath.begin(), lockRootAttrPath.end()); input.follows = follows; } else @@ -264,7 +264,7 @@ static Flake readFlake( if (auto description = vInfo.attrs()->get(state.s.description)) { expectType(state, nString, *description->value, description->pos); - flake.description = description->value->c_str(); + flake.description = description->value->string_view(); } auto sInputs = state.symbols.create("inputs"); diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 3903823aa..5934e8479 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -153,9 +153,9 @@ nix_err nix_err_code(const nix_c_context * read_context) } // internal -nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data) +nix_err call_nix_get_string_callback(const std::string_view str, nix_get_string_callback callback, void * user_data) { - callback(str.c_str(), str.size(), user_data); + callback(str.data(), str.size(), user_data); return NIX_OK; } diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h index 92bb9c1d2..e4c5e93bb 100644 --- a/src/libutil-c/nix_api_util_internal.h +++ b/src/libutil-c/nix_api_util_internal.h @@ -32,7 +32,7 @@ nix_err nix_context_error(nix_c_context * context); * @return NIX_OK if there were no errors. * @see nix_get_string_callback */ -nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data); +nix_err call_nix_get_string_callback(const std::string_view str, nix_get_string_callback callback, void * user_data); #define NIXC_CATCH_ERRS \ catch (...) \ diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index 01c8ccf4b..2a0984d18 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -1228,7 +1228,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->c_str(); + attrs2["value"] = v->string_view(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; @@ -1249,7 +1249,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->c_str(); + attrs3["value"] = elem->string_view(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { @@ -1260,7 +1260,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = i.value->c_str(); + attrs3["value"] = i.value->string_view(); xml.writeEmptyElement("string", attrs3); } } From 2f6c865e25ee41ec2ba5b8f087a29512ad7aff82 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Nov 2025 13:22:28 +0100 Subject: [PATCH 174/201] getAccessorFromCommit(): Remove superfluous infoAttrs variable --- src/libfetchers/git.cc | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c8311c17f..710d2f315 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -735,13 +735,10 @@ struct GitInputScheme : InputScheme auto rev = *input.getRev(); - Attrs infoAttrs({ - {"rev", rev.gitRev()}, - {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)}, - }); + input.attrs.insert_or_assign("lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)); if (!getShallowAttr(input)) - infoAttrs.insert_or_assign("revCount", getRevCount(*input.settings, repoInfo, repoDir, rev)); + input.attrs.insert_or_assign("revCount", getRevCount(*input.settings, repoInfo, repoDir, rev)); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -797,9 +794,6 @@ struct GitInputScheme : InputScheme } assert(!origRev || origRev == rev); - if (!getShallowAttr(input)) - input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return {accessor, std::move(input)}; } From 4a0ccc89d9721fd41dc66f74b475f39df60ed20f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Nov 2025 13:58:23 +0100 Subject: [PATCH 175/201] ThreadPool::enqueue(): Use move semantics This avoids a superfluous copy of the work item. --- src/libutil/include/nix/util/thread-pool.hh | 2 +- src/libutil/thread-pool.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/include/nix/util/thread-pool.hh b/src/libutil/include/nix/util/thread-pool.hh index 811c03d88..a07354146 100644 --- a/src/libutil/include/nix/util/thread-pool.hh +++ b/src/libutil/include/nix/util/thread-pool.hh @@ -36,7 +36,7 @@ public: /** * Enqueue a function to be executed by the thread pool. */ - void enqueue(const work_t & t); + void enqueue(work_t t); /** * Execute work items until the queue is empty. diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index b7740bc3e..24bdeef86 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -41,12 +41,12 @@ void ThreadPool::shutdown() thr.join(); } -void ThreadPool::enqueue(const work_t & t) +void ThreadPool::enqueue(work_t t) { auto state(state_.lock()); if (quit) throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); - state->pending.push(t); + state->pending.push(std::move(t)); /* Note: process() also executes items, so count it as a worker. */ if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads) state->workers.emplace_back(&ThreadPool::doWork, this, false); From 53b4ea6c85e2d000b8badc923066866ba8de324c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 27 Oct 2025 11:26:46 -0700 Subject: [PATCH 176/201] Add documentation for NAR spec in kaitai * Add a new flake check * Add unit tests * Add Kaitai spec * Updated documentation --- doc/manual/source/SUMMARY.md.in | 2 +- .../{nix-archive.md => nix-archive/index.md} | 12 ++ .../source/protocols/nix-archive/nar.ksy | 169 ++++++++++++++++++ .../file-system-object/content-address.md | 2 +- flake.nix | 4 + meson.build | 1 + packaging/components.nix | 5 + packaging/dev-shell.nix | 4 +- packaging/hydra.nix | 1 + src/kaitai-struct-checks/.version | 1 + src/kaitai-struct-checks/meson.build | 77 ++++++++ src/kaitai-struct-checks/nar.ksy | 1 + src/kaitai-struct-checks/nars | 1 + .../nix-meson-build-support | 1 + src/kaitai-struct-checks/package.nix | 75 ++++++++ src/kaitai-struct-checks/test-parse-nar.cc | 48 +++++ src/nix/nar.md | 2 +- 17 files changed, 402 insertions(+), 4 deletions(-) rename doc/manual/source/protocols/{nix-archive.md => nix-archive/index.md} (73%) create mode 100644 doc/manual/source/protocols/nix-archive/nar.ksy create mode 120000 src/kaitai-struct-checks/.version create mode 100644 src/kaitai-struct-checks/meson.build create mode 120000 src/kaitai-struct-checks/nar.ksy create mode 120000 src/kaitai-struct-checks/nars create mode 120000 src/kaitai-struct-checks/nix-meson-build-support create mode 100644 src/kaitai-struct-checks/package.nix create mode 100644 src/kaitai-struct-checks/test-parse-nar.cc diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 7f3b1a103..287dff872 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -125,7 +125,7 @@ - [Deriving Path](protocols/json/deriving-path.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - - [Nix Archive (NAR) Format](protocols/nix-archive.md) + - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [C API](c-api.md) - [Glossary](glossary.md) diff --git a/doc/manual/source/protocols/nix-archive.md b/doc/manual/source/protocols/nix-archive/index.md similarity index 73% rename from doc/manual/source/protocols/nix-archive.md rename to doc/manual/source/protocols/nix-archive/index.md index 02a8dd464..4d25f63e2 100644 --- a/doc/manual/source/protocols/nix-archive.md +++ b/doc/manual/source/protocols/nix-archive/index.md @@ -41,3 +41,15 @@ The `str` function / parameterized rule is defined as follows: - `int(n)` = the 64-bit little endian representation of the number `n` - `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte + +## Kaitai Struct Specification + +The Nix Archive (NAR) format is also formally described using [Kaitai Struct](https://kaitai.io/), an Interface Description Language (IDL) for defining binary data structures. + +> Kaitai Struct provides a language-agnostic, machine-readable specification that can be compiled into parsers for various programming languages (e.g., C++, Python, Java, Rust). + +```yaml +{{#include nar.ksy}} +``` + +The source of the spec can be found [here](https://github.com/nixos/nix/blob/master/src/nix-manual/source/protocols/nix-archive/nar.ksy). Contributions and improvements to the spec are welcomed. \ No newline at end of file diff --git a/doc/manual/source/protocols/nix-archive/nar.ksy b/doc/manual/source/protocols/nix-archive/nar.ksy new file mode 100644 index 000000000..1cad09097 --- /dev/null +++ b/doc/manual/source/protocols/nix-archive/nar.ksy @@ -0,0 +1,169 @@ +meta: + id: nix_nar + title: Nix Archive (NAR) + file-extension: nar + endian: le +doc: | + Nix Archive (NAR) format. A simple, reproducible binary archive + format used by the Nix package manager to serialize file system objects. +doc-ref: 'https://nixos.org/manual/nix/stable/command-ref/nix-store.html#nar-format' + +seq: + - id: magic + type: padded_str + doc: "Magic string, must be 'nix-archive-1'." + valid: + expr: _.body == 'nix-archive-1' + - id: root_node + type: node + doc: "The root of the archive, which is always a single node." + +types: + padded_str: + doc: | + A string, prefixed with its length (u8le) and + padded with null bytes to the next 8-byte boundary. + seq: + - id: len_str + type: u8 + - id: body + type: str + size: len_str + encoding: 'ascii' + - id: padding + size: (8 - (len_str % 8)) % 8 + + node: + doc: "A single filesystem node (file, directory, or symlink)." + seq: + - id: open_paren + type: padded_str + doc: "Must be '(', a token starting the node definition." + valid: + expr: _.body == '(' + - id: type_key + type: padded_str + doc: "Must be 'type'." + valid: + expr: _.body == 'type' + - id: type_val + type: padded_str + doc: "The type of the node: 'regular', 'directory', or 'symlink'." + - id: body + type: + switch-on: type_val.body + cases: + "'directory'": type_directory + "'regular'": type_regular + "'symlink'": type_symlink + - id: close_paren + type: padded_str + valid: + expr: _.body == ')' + if: "type_val.body != 'directory'" + doc: "Must be ')', a token ending the node definition." + + type_directory: + doc: "A directory node, containing a list of entries. Entries must be ordered by their names." + seq: + - id: entries + type: dir_entry + repeat: until + repeat-until: _.kind.body == ')' + types: + dir_entry: + doc: "A single entry within a directory, or a terminator." + seq: + - id: kind + type: padded_str + valid: + expr: _.body == 'entry' or _.body == ')' + doc: "Must be 'entry' (for a child node) or '' (for terminator)." + - id: open_paren + type: padded_str + valid: + expr: _.body == '(' + if: 'kind.body == "entry"' + - id: name_key + type: padded_str + valid: + expr: _.body == 'name' + if: 'kind.body == "entry"' + - id: name + type: padded_str + if: 'kind.body == "entry"' + - id: node_key + type: padded_str + valid: + expr: _.body == 'node' + if: 'kind.body == "entry"' + - id: node + type: node + if: 'kind.body == "entry"' + doc: "The child node, present only if kind is 'entry'." + - id: close_paren + type: padded_str + valid: + expr: _.body == ')' + if: 'kind.body == "entry"' + instances: + is_terminator: + value: kind.body == ')' + + type_regular: + doc: "A regular file node." + seq: + # Read attributes (like 'executable') until we hit 'contents' + - id: attributes + type: reg_attribute + repeat: until + repeat-until: _.key.body == "contents" + # After the 'contents' token, read the file data + - id: file_data + type: file_content + instances: + is_executable: + value: 'attributes[0].key.body == "executable"' + doc: "True if the file has the 'executable' attribute." + types: + reg_attribute: + doc: "An attribute of the file, e.g., 'executable' or 'contents'." + seq: + - id: key + type: padded_str + doc: "Attribute key, e.g., 'executable' or 'contents'." + valid: + expr: _.body == 'executable' or _.body == 'contents' + - id: value + type: padded_str + if: 'key.body == "executable"' + valid: + expr: _.body == '' + doc: "Must be '' if key is 'executable'." + file_content: + doc: "The raw data of the file, prefixed by length." + seq: + - id: len_contents + type: u8 + # # This relies on the property of instances that they are lazily evaluated and cached. + - size: 0 + if: nar_offset < 0 + - id: contents + size: len_contents + - id: padding + size: (8 - (len_contents % 8)) % 8 + instances: + nar_offset: + value: _io.pos + + type_symlink: + doc: "A symbolic link node." + seq: + - id: target_key + type: padded_str + doc: "Must be 'target'." + valid: + expr: _.body == 'target' + - id: target_val + type: padded_str + doc: "The destination path of the symlink." diff --git a/doc/manual/source/store/file-system-object/content-address.md b/doc/manual/source/store/file-system-object/content-address.md index 04a1021f1..5685de03e 100644 --- a/doc/manual/source/store/file-system-object/content-address.md +++ b/doc/manual/source/store/file-system-object/content-address.md @@ -46,7 +46,7 @@ be many different serialisations. For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, which is carefully designed to avoid the problems described above. -The exact specification of the Nix Archive format is in [specified here](../../protocols/nix-archive.md). +The exact specification of the Nix Archive format is in [specified here](../../protocols/nix-archive/index.md). ## Content addressing File System Objects beyond a single serialisation pass diff --git a/flake.nix b/flake.nix index e25722d46..a70617b74 100644 --- a/flake.nix +++ b/flake.nix @@ -417,6 +417,10 @@ supportsCross = false; }; + "nix-kaitai-struct-checks" = { + supportsCross = false; + }; + "nix-perl-bindings" = { supportsCross = false; }; diff --git a/meson.build b/meson.build index f3158ea6d..c493dfad6 100644 --- a/meson.build +++ b/meson.build @@ -61,3 +61,4 @@ if get_option('unit-tests') endif subproject('nix-functional-tests') subproject('json-schema-checks') +subproject('kaitai-struct-checks') diff --git a/packaging/components.nix b/packaging/components.nix index f9d7b109a..bbd6208b9 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -443,6 +443,11 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; + /** + Kaitai struct schema validation checks + */ + nix-kaitai-struct-checks = callPackage ../src/kaitai-struct-checks/package.nix { }; + nix-perl-bindings = callPackage ../src/perl/package.nix { }; /** diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 153e7a3eb..ea12e079f 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -109,6 +109,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs ++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs + ++ pkgs.nixComponents2.nix-kaitai-struct-checks.externalNativeBuildInputs ++ lib.optional ( !buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 @@ -148,6 +149,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ++ pkgs.nixComponents2.nix-expr.externalPropagatedBuildInputs ++ pkgs.nixComponents2.nix-cmd.buildInputs ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.externalBuildInputs - ++ lib.optional havePerl pkgs.perl; + ++ lib.optional havePerl pkgs.perl + ++ pkgs.nixComponents2.nix-kaitai-struct-checks.externalBuildInputs; } ) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 3bbb6c15b..67e2c0dfd 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -63,6 +63,7 @@ let "nix-cli" "nix-functional-tests" "nix-json-schema-checks" + "nix-kaitai-struct-checks" ] ++ lib.optionals enableBindings [ "nix-perl-bindings" diff --git a/src/kaitai-struct-checks/.version b/src/kaitai-struct-checks/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/kaitai-struct-checks/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/kaitai-struct-checks/meson.build b/src/kaitai-struct-checks/meson.build new file mode 100644 index 000000000..f705a6744 --- /dev/null +++ b/src/kaitai-struct-checks/meson.build @@ -0,0 +1,77 @@ +# Run with: +# meson test --suite kaitai-struct +# Run with: (without shell / configure) +# nix build .#nix-kaitai-struct-checks + +project( + 'nix-kaitai-struct-checks', + 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++23', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +kaitai_runtime_dep = dependency('kaitai-struct-cpp-stl-runtime', required : true) +gtest_dep = dependency('gtest') +gtest_main_dep = dependency('gtest_main', required : true) + +# Find the Kaitai Struct compiler +ksc = find_program('ksc', required : true) + +kaitai_generated_srcs = custom_target( + 'kaitai-generated-sources', + input : [ 'nar.ksy' ], + output : [ 'nix_nar.cpp', 'nix_nar.h' ], + command : [ + ksc, + '@INPUT@', + '--target', 'cpp_stl', + '--outdir', + meson.current_build_dir(), + ], +) + +nar_kaitai_lib = library( + 'nix-nar-kaitai-lib', + kaitai_generated_srcs, + dependencies : [ kaitai_runtime_dep ], + install : true, +) + +nar_kaitai_dep = declare_dependency( + link_with : nar_kaitai_lib, + sources : kaitai_generated_srcs[1], +) + +# The nar directory is a committed symlink to the actual nars location +nars_dir = meson.current_source_dir() / 'nars' + +# Get all example files +nars = [ + 'dot.nar', +] + +test_deps = [ + nar_kaitai_dep, + kaitai_runtime_dep, + gtest_main_dep, +] + +this_exe = executable( + meson.project_name(), + 'test-parse-nar.cc', + dependencies : test_deps, +) + +test( + meson.project_name(), + this_exe, + env : [ 'NIX_NARS_DIR=' + nars_dir ], + protocol : 'gtest', +) diff --git a/src/kaitai-struct-checks/nar.ksy b/src/kaitai-struct-checks/nar.ksy new file mode 120000 index 000000000..c3a79a3b6 --- /dev/null +++ b/src/kaitai-struct-checks/nar.ksy @@ -0,0 +1 @@ +../../doc/manual/source/protocols/nix-archive/nar.ksy \ No newline at end of file diff --git a/src/kaitai-struct-checks/nars b/src/kaitai-struct-checks/nars new file mode 120000 index 000000000..ed0b4ecc7 --- /dev/null +++ b/src/kaitai-struct-checks/nars @@ -0,0 +1 @@ +../libutil-tests/data/nars \ No newline at end of file diff --git a/src/kaitai-struct-checks/nix-meson-build-support b/src/kaitai-struct-checks/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/kaitai-struct-checks/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/kaitai-struct-checks/package.nix b/src/kaitai-struct-checks/package.nix new file mode 100644 index 000000000..263dd6fd1 --- /dev/null +++ b/src/kaitai-struct-checks/package.nix @@ -0,0 +1,75 @@ +# Run with: nix build .#nix-kaitai-struct-checks +{ + lib, + mkMesonDerivation, + gtest, + meson, + ninja, + pkg-config, + kaitai-struct-compiler, + fetchzip, + kaitai-struct-cpp-stl-runtime, + # Configuration Options + version, +}: +let + inherit (lib) fileset; +in +mkMesonDerivation (finalAttrs: { + pname = "nix-kaitai-struct-checks"; + inherit version; + + workDir = ./.; + fileset = lib.fileset.unions [ + ../../nix-meson-build-support + ./nix-meson-build-support + ./.version + ../../.version + ../../doc/manual/source/protocols/nix-archive/nar.ksy + ./nars + ../../src/libutil-tests/data + ./meson.build + ./nar.ksy + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" ]; + + passthru.externalNativeBuildInputs = [ + # This can go away when we bump up to 25.11 + (kaitai-struct-compiler.overrideAttrs (finalAttrs: { + version = "0.11"; + src = fetchzip { + url = "https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/${version}/kaitai-struct-compiler-${version}.zip"; + sha256 = "sha256-j9TEilijqgIiD0GbJfGKkU1FLio9aTopIi1v8QT1b+A="; + }; + })) + ]; + + passthru.externalBuildInputs = [ + gtest + kaitai-struct-cpp-stl-runtime + ]; + + buildInputs = finalAttrs.passthru.externalBuildInputs; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ] + ++ finalAttrs.passthru.externalNativeBuildInputs; + + doCheck = true; + + mesonCheckFlags = [ "--print-errorlogs" ]; + + postInstall = '' + touch $out + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/kaitai-struct-checks/test-parse-nar.cc b/src/kaitai-struct-checks/test-parse-nar.cc new file mode 100644 index 000000000..456ffb127 --- /dev/null +++ b/src/kaitai-struct-checks/test-parse-nar.cc @@ -0,0 +1,48 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "nix_nar.h" + +static const std::vector NarFiles = { + "empty.nar", + "dot.nar", + "dotdot.nar", + "executable-after-contents.nar", + "invalid-tag-instead-of-contents.nar", + "name-after-node.nar", + "nul-character.nar", + "slash.nar", +}; + +class NarParseTest : public ::testing::TestWithParam +{}; + +TEST_P(NarParseTest, ParseSucceeds) +{ + const auto nar_file = GetParam(); + + const char * nars_dir_env = std::getenv("NIX_NARS_DIR"); + if (nars_dir_env == nullptr) { + FAIL() << "NIX_NARS_DIR environment variable not set."; + } + + const std::filesystem::path nar_file_path = std::filesystem::path(nars_dir_env) / "dot.nar"; + ASSERT_TRUE(std::filesystem::exists(nar_file_path)) << "Missing test file: " << nar_file_path; + + std::ifstream ifs(nar_file_path, std::ifstream::binary); + ASSERT_TRUE(ifs.is_open()) << "Failed to open file: " << nar_file; + kaitai::kstream ks(&ifs); + nix_nar_t nar(&ks); + ASSERT_TRUE(nar.root_node() != nullptr) << "Failed to parse NAR file: " << nar_file; +} + +INSTANTIATE_TEST_SUITE_P(AllNarFiles, NarParseTest, ::testing::ValuesIn(NarFiles)); diff --git a/src/nix/nar.md b/src/nix/nar.md index b0f70ce93..c29c2092a 100644 --- a/src/nix/nar.md +++ b/src/nix/nar.md @@ -8,7 +8,7 @@ R""( # File format For the definition of the Nix Archive file format, see -[within the protocols chapter](@docroot@/protocols/nix-archive.md) +[within the protocols chapter](@docroot@/protocols/nix-archive/index.md) of the manual. [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive From 72d0f7b61941225eb06762095131f2b42cd3a56a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 29 Oct 2025 02:16:50 -0400 Subject: [PATCH 177/201] Document "hash derivation quotiented", resolution, and build trace Progress on #13405, which asks for an explicit characterisation of the equivalence relation like the one given here. Also progress on #11895, because we're using the term "build trace entry" instead of "realisation". Mention #9259, a future work item. Co-authored-by: Robert Hensing --- doc/manual/book.toml.in | 1 + doc/manual/meson.build | 2 + doc/manual/source/SUMMARY.md.in | 3 + .../source/protocols/derivation-aterm.md | 4 +- .../protocols/json/schema/derivation-v3.yaml | 4 +- doc/manual/source/store/build-trace.md | 53 +++++ doc/manual/source/store/derivation/index.md | 2 +- .../derivation/outputs/content-address.md | 4 +- .../store/derivation/outputs/input-address.md | 225 ++++++++++++++++-- doc/manual/source/store/math-notation.md | 16 ++ doc/manual/source/store/resolution.md | 219 +++++++++++++++++ doc/manual/theme/head.hbs | 15 ++ src/libstore/include/nix/store/derivations.hh | 2 +- 13 files changed, 528 insertions(+), 22 deletions(-) create mode 100644 doc/manual/source/store/build-trace.md create mode 100644 doc/manual/source/store/math-notation.md create mode 100644 doc/manual/source/store/resolution.md create mode 100644 doc/manual/theme/head.hbs diff --git a/doc/manual/book.toml.in b/doc/manual/book.toml.in index 34acf642e..bacca59ff 100644 --- a/doc/manual/book.toml.in +++ b/doc/manual/book.toml.in @@ -7,6 +7,7 @@ additional-css = ["custom.css"] additional-js = ["redirects.js"] edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" git-repository-url = "https://github.com/NixOS/nix" +mathjax-support = true # Handles replacing @docroot@ with a path to ./source relative to that markdown file, # {{#include handlebars}}, and the @generated@ syntax used within these. it mostly diff --git a/doc/manual/meson.build b/doc/manual/meson.build index fdea40098..231f7b9f8 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -92,6 +92,8 @@ manual = custom_target( (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 rm -rf @2@/manual mv @2@/html @2@/manual + # Remove Mathjax 2.7, because we will actually use MathJax 3.x + find @2@/manual | grep .html | xargs sed -i -e '/2.7.1.MathJax.js/d' find @2@/manual -iname meson.build -delete '''.format( python.full_path(), diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 287dff872..b87bf93a3 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -26,9 +26,12 @@ - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) - [Input-addressing derivation outputs](store/derivation/outputs/input-address.md) + - [Build Trace](store/build-trace.md) + - [Derivation Resolution](store/resolution.md) - [Building](store/building.md) - [Store Types](store/types/index.md) {{#include ./store/types/SUMMARY.md}} + - [Appendix: Math notation](store/math-notation.md) - [Nix Language](language/index.md) - [Data Types](language/types.md) - [String context](language/string-context.md) diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 99e3c2be6..523678e66 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -1,6 +1,8 @@ # Derivation "ATerm" file format -For historical reasons, [store derivations][store derivation] are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. +For historical reasons, [store derivations][store derivation] are stored on-disk in "Annotated Term" (ATerm) format +([guide](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html), +[paper](https://doi.org/10.1002/(SICI)1097-024X(200003)30:3%3C259::AID-SPE298%3E3.0.CO;2-Y)). ## The ATerm format used diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index 9c0210bb7..fa68adcb1 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -39,9 +39,9 @@ properties: This is a guard that allows us to continue evolving this format. The choice of `3` is fairly arbitrary, but corresponds to this informal version: - - Version 0: A-Term format + - Version 0: ATerm format - - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format. + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - Version 2: Separate `method` and `hashAlgo` fields in output specs diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md new file mode 100644 index 000000000..1086dcb88 --- /dev/null +++ b/doc/manual/source/store/build-trace.md @@ -0,0 +1,53 @@ +# Build Trace + +> **Warning** +> +> This entire concept is currently +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) +> and subject to change. + +The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. +It maps the inputs of builds to the outputs of builds. +Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. + +In general the derivations used as a key should be [*resolved*](./resolution.md). +A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. +If all the resolved inputs of a derivation are content-addressed, that means the inputs will be fully determined, leaving no ambiguity for what build was performed. +(Input-addressed inputs however are still ambiguous. They too should be locked down, but this is left as future work.) + +Accordingly, to look up an unresolved derivation, one must first resolve it to get a resolved derivation. +Resolving itself involves looking up entries in the build trace, so this is a mutually recursive process that will end up inspecting possibly many entries. + +Except for the issue with input-addressed paths called out above, base build traces are trivially *coherent* -- incoherence is not possible. +That means that the claims that each key-value base build try entry makes are independent, and no mapping invalidates another mapping. + +Whether the mappings are *true*, i.e. the faithful recording of actual builds performed, is another matter. +Coherence is about the multiple claims of the build trace being mutually consistent, not about whether the claims are individually true or false. + +In general, there is no way to audit a build trace entry except for by performing the build again from scratch. +And even in that case, a different result doesn't mean the original entry was a "lie", because the derivation being built may be non-deterministic. +As such, the decision of whether to trust a counterparty's build trace is a fundamentally subject policy choice. +Build trace entries are typically *signed* in order to enable arbitrary public-key-based trust polices. + +## Derived build traces + +Implementations that wish to memoize the above may also keep additional *derived* build trace entries that do map unresolved derivations. +But if they do so, they *must* also keep the underlying base entries with resolved derivation keys around. +Firstly, this ensures that the derived entries are merely cache, which could be recomputed from scratch. +Secondly, this ensures the coherence of the derived build trace. + +Unlike with base build traces, incoherence with derived build traces is possible. +The key ingredient is that derivation resolution is only deterministic with respect to a fixed base build trace. +Without fixing the base build trace, it inherits the subjectivity of base build traces themselves. + +Concretely, suppose there are three derivations \\(a\\), \\(b\\), and \((c\\). +Let \\(a\\) be a resolved derivation, but let \\(b\\) and \((c\\) be unresolved and both take as an input an output of \\(a\\). +Now suppose that derived entries are made for \\(b\\) and \((c\\) based on two different entries of \\(a\\). +(This could happen if \\(a\\) is non-deterministic, \\(a\\) and \\(b\\) are built in one store, \\(a\\) and \\(c\\) are built in another store, and then a third store substitutes from both of the first two stores.) + +If trusting the derived build trace entries for \\(b\\) and \((c\\) requires that each's underlying entry for \\(a\\) be also trusted, the two different mappings for \\(a\\) will be caught. +However, if \\(b\\) and \((c\\)'s entries can be combined in isolation, there will be nothing to catch the contradiction in their hidden assumptions about \\(a\\)'s output. + +[derivation]: ./derivation/index.md +[output]: ./derivation/outputs/index.md +[store object]: @docroot@/store/store-object.md diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 5b179273d..61c5335ff 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -245,7 +245,7 @@ If those other derivations *also* abide by this common case (and likewise for tr > note the ".drv" > ``` -## Extending the model to be higher-order +## Extending the model to be higher-order {#dynamic} **Experimental feature**: [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index 4d5130348..aa65fbe49 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -167,10 +167,10 @@ It is only in the potential for that check to fail that they are different. > > In a future world where floating content-addressing is also stable, we in principle no longer need separate [fixed](#fixed) content-addressing. > Instead, we could always use floating content-addressing, and separately assert the precise value content address of a given store object to be used as an input (of another derivation). -> A stand-alone assertion object of this sort is not yet implemented, but its possible creation is tracked in [Issue #11955](https://github.com/NixOS/nix/issues/11955). +> A stand-alone assertion object of this sort is not yet implemented, but its possible creation is tracked in [issue #11955](https://github.com/NixOS/nix/issues/11955). > > In the current version of Nix, fixed outputs which fail their hash check are still registered as valid store objects, just not registered as outputs of the derivation which produced them. -> This is an optimization that means if the wrong output hash is specified in a derivation, and then the derivation is recreated with the right output hash, derivation does not need to be rebuilt --- avoiding downloading potentially large amounts of data twice. +> This is an optimization that means if the wrong output hash is specified in a derivation, and then the derivation is recreated with the right output hash, derivation does not need to be rebuilt — avoiding downloading potentially large amounts of data twice. > This optimisation prefigures the design above: > If the output hash assertion was removed outside the derivation itself, Nix could additionally not only register that outputted store object like today, but could also make note that derivation did in fact successfully download some data. For example, for the "fetch URL" example above, making such a note is tantamount to recording what data is available at the time of download at the given URL. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index e2e15a801..3fd20f17d 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -6,26 +6,221 @@ That is to say, an input-addressed output's store path is a function not of the output itself, but of the derivation that produced it. Even if two store paths have the same contents, if they are produced in different ways, and one is input-addressed, then they will have different store paths, and thus guaranteed to not be the same store object. - +type FirstOrderDerivingPath = ConstantPath | FirstOrderOutputPath; +type Inputs = Set; +``` + +For the algorithm below, we adopt a derivation where the two types of (first order) derived paths are partitioned into two sets, as follows: +```typescript +type Derivation = { + // inputs: Set; // replaced + inputSrcs: Set; // new instead + inputDrvOutputs: Set; // new instead + // ...other fields... +}; +``` + +In the [currently-experimental][xp-feature-dynamic-derivations] higher-order case where outputs of outputs are allowed as [deriving paths][deriving-path] and thus derivation inputs, derivations using that generalization are not valid arguments to this function. +Those derivations must be (partially) [resolved](@docroot@/store/resolution.md) enough first, to the point where no such higher-order inputs remain. +Then, and only then, can input addresses be assigned. + +``` +function hashQuotientDerivation(drv) -> Hash: + assert(drv.outputs are input-addressed) + drv′ ← drv with { + inputDrvOutputs = ⋃( + assert(drvPath is store path) + case hashOutputsOrQuotientDerivation(readDrv(drvPath)) of + drvHash : Hash → + (drvHash.toBase16(), output) + outputHashes : Map[String, Hash] → + (outputHashes[output].toBase16(), "out") + | (drvPath, output) ∈ drv.inputDrvOutputs + ) + } + return hashSHA256(printDrv(drv′)) + +function hashOutputsOrQuotientDerivation(drv) -> Map[String, Hash] | Hash: + if drv.outputs are content-addressed: + return { + outputName ↦ hashSHA256( + "fixed:out:" + ca.printMethodAlgo() + + ":" + ca.hash.toBase16() + + ":" + ca.makeFixedOutputPath(drv.name, outputName)) + | (outputName ↦ output) ∈ drv.outputs + , ca = output.contentAddress // or get from build trace if floating + } + else: // drv.outputs are input-addressed + return hashQuotientDerivation(drv) +``` + +### `hashQuotientDerivation` + +We replace each element in the derivation's `inputDrvOutputs` using data from a call to `hashOutputsOrQuotientDerivation` on the `drvPath` of that element. +When `hashOutputsOrQuotientDerivation` returns a single drv hash (because the input derivation in question is input-addressing), we simply swap out the `drvPath` for that hash, and keep the same output name. +When `hashOutputsOrQuotientDerivation` returns a map of content addresses per-output, we look up the output in question, and pair it with the output name `out`. + +The resulting pseudo-derivation (with hashes instead of store paths in `inputDrvs`) is then printed (in the ["ATerm" format](@docroot@/protocols/derivation-aterm.md)) and hashed, and this becomes the hash of the "quotient derivation". + +When calculating output hashes, `hashQuotientDerivation` is called on an almost-complete input-addressing derivation, which is just missing its input-addressed outputs paths. +The derivation hash is then used to calculate output paths for each output. + +Those output paths can then be substituted into the almost-complete input-addressed derivation to complete it. + +> **Note** +> +> There may be an unintentional deviation from specification currently implemented in the `(outputHashes[output].toBase16(), "out")` case. +> This is not fatal because the deviation would only apply for content-addressing derivations with more than one output, and that only occurs in the floating case, which is [experimental][xp-feature-ca-derivations]. +> Once this bug is fixed, this note will be removed. + +### `hashOutputsOrQuotientDerivation` + +How does `hashOutputsOrQuotientDerivation` in turn work? +It consists of two main cases, based on whether the outputs of the derivation are to be input-addressed or content-addressed. + +#### Input-addressed outputs case + +In the input-addressed case, it just calls `hashQuotientDerivation`, and returns that derivation hash. +This makes `hashQuotientDerivation` and `hashOutputsOrQuotientDerivation` mutually-recursive. + +> **Note** +> +> In this case, `hashQuotientDerivation` is being called on a *complete* input-addressing derivation that already has its output paths calculated. +> The `inputDrvs` substitution takes place anyways. + +#### Content-addressed outputs case + +If the outputs are [content-addressed](./content-address.md), then it computes a hash for each output derived from the content-address of that output. + +> **Note** +> +> In the [fixed](./content-address.md#fixed) content-addressing case, the outputs' content addresses are statically specified in advance, so this always just works. +> (The fixed case is what the pseudo-code shows.) +> +> In the [floating](./content-address.md#floating) case, the content addresses are not specified in advance. +> This is what the "or get from [build trace](@docroot@/store/build-trace.md) if floating" comment refers to. +> In this case, the algorithm is *stuck* until the input in question is built, and we know what the actual contents of the output in question is. +> +> That is OK however, because there is no problem with delaying the assigning of input addresses (which, remember, is what `hashQuotientDerivation` is ultimately for) until all inputs are known. + +### Performance + +The recursion in the algorithm is potentially inefficient: +it could call itself once for each path by which a subderivation can be reached, i.e., `O(V^k)` times for a derivation graph with `V` derivations and with out-degree of at most `k`. +In the actual implementation, [memoisation](https://en.wikipedia.org/wiki/Memoization) is used to reduce this cost to be proportional to the total number of `inputDrvOutputs` encountered. + +### Semantic properties + +*See [this chapter's appendix](@docroot@/store/math-notation.md) on grammar and metavariable conventions.* + +In essence, `hashQuotientDerivation` partitions input-addressing derivations into equivalence classes: every derivation in that equivalence class is mapped to the same derivation hash. +We can characterize this equivalence relation directly, by working bottom up. + +We start by defining an equivalence relation on first-order output deriving paths that refer content-addressed derivation outputs. Two such paths are equivalent if they refer to the same store object: + +\\[ +\\begin{prooftree} +\\AxiomC{$d\_1$ is content-addressing} +\\AxiomC{$d\_2$ is content-addressing} +\\AxiomC{$ + {}^\*(\text{path}(d\_1), o\_1) + \= + {}^\*(\text{path}(d\_2), o\_2) +$} +\\TrinaryInfC{$(\text{path}(d\_1), o\_1) \\,\\sim_{\\mathrm{CA}}\\, (d\_2, o\_2)$} +\\end{prooftree} +\\] + +where \\({}^*(s, o)\\) denotes the store object that the output deriving path refers to. + +We will also need the following construction to lift any equivalence relation on \\(X\\) to an equivalence relation on (finite) sets of \\(X\\) (in short, \\(\\mathcal{P}(X)\\)): + +\\[ +\\begin{prooftree} +\\AxiomC{$\\forall a \\in A. \\exists b \\in B. a \\,\\sim\_X\\, b$} +\\AxiomC{$\\forall b \\in B. \\exists a \\in A. b \\,\\sim\_X\\, a$} +\\BinaryInfC{$A \\,\\sim_{\\mathcal{P}(X)}\\, B$} +\\end{prooftree} +\\] + +Now we can define the equivalence relation \\(\\sim_\\mathrm{IA}\\) on input-addressed derivation outputs. Two input-addressed outputs are equivalent if their derivations are equivalent (via the yet-to-be-defined \\(\\sim_{\\mathrm{IADrv}}\\) relation) and their output names are the same: + +\\[ +\\begin{prooftree} +\\AxiomC{$d\_1$ is input-addressing} +\\AxiomC{$d\_2$ is input-addressing} +\\AxiomC{$d\_1 \\,\\sim_{\\mathrm{IADrv}}\\, d\_2$} +\\AxiomC{$o\_1 = o\_2$} +\\QuaternaryInfC{$(\text{path}(d\_1), o\_1) \\,\\sim_{\\mathrm{IA}}\\, (\text{path}(d\_2), o\_2)$} +\\end{prooftree} +\\] + +And now we can define \\(\\sim_{\\mathrm{IADrv}}\\). +Two input-addressed derivations are equivalent if their content-addressed inputs are equivalent, their input-addressed inputs are also equivalent, and they are otherwise equal: + + + +\\[ +\\begin{prooftree} +\\alwaysNoLine +\\AxiomC{$ + \\mathrm{caInputs}(d\_1) + \\,\\sim_{\\mathcal{P}(\\mathrm{CA})}\\, + \\mathrm{caInputs}(d\_2) +$} +\\AxiomC{$ + \\mathrm{iaInputs}(d\_1) + \\,\\sim_{\\mathcal{P}(\\mathrm{IA})}\\, + \\mathrm{iaInputs}(d\_2) +$} +\\BinaryInfC{$ + d\_1\left[\\mathrm{inputDrvOutputs} := \\{\\}\right] + \= + d\_2\left[\\mathrm{inputDrvOutputs} := \\{\\}\right] +$} +\\alwaysSingleLine +\\UnaryInfC{$d\_1 \\,\\sim_{\\mathrm{IADrv}}\\, d\_2$} +\\end{prooftree} +\\] + +where \\(\\mathrm{caInputs}(d)\\) returns the content-addressed inputs of \\(d\\) and \\(\\mathrm{iaInputs}(d)\\) returns the input-addressed inputs. + +> **Note** +> +> An astute reader might notice that that nowhere does `inputSrcs` enter into these definitions. +> That means that replacing an input derivation with its outputs directly added to `inputSrcs` always results in a derivation in a different equivalence class, despite the resulting input closure (as would be mounted in the store at build time) being the same. +> [Issue #9259](https://github.com/NixOS/nix/issues/9259) is about creating a coarser equivalence relation to address this. +> +> \\(\\sim_\mathrm{Drv}\\) from [derivation resolution](@docroot@/store/resolution.md) is such an equivalence relation. +> It is coarser than this one: any two derivations which are "'hash quotient derivation'-equivalent" (\\(\\sim_\mathrm{IADrv}\\)) are also "resolution-equivalent" (\\(\\sim_\mathrm{Drv}\\)). +> It also relates derivations whose `inputDrvOutputs` have been rewritten into `inputSrcs`. + +[deriving-path]: @docroot@/store/derivation/index.md#deriving-path +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations [xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations -[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing -[xp-feature-impure-derivations]: @docroot@/development/experimental-features.md#xp-feature-impure-derivations diff --git a/doc/manual/source/store/math-notation.md b/doc/manual/source/store/math-notation.md new file mode 100644 index 000000000..723982e73 --- /dev/null +++ b/doc/manual/source/store/math-notation.md @@ -0,0 +1,16 @@ +# Appendix: Math notation + +A few times in this manual, formal "proof trees" are used for [natural deduction](https://en.wikipedia.org/wiki/Natural_deduction)-style definition of various [relations](https://en.wikipedia.org/wiki/Relation_(mathematics)). + +The following grammar and assignment of metavariables to syntactic categories is used in these sections. + +\\begin{align} +s, t &\in \text{store-path} \\\\ +o &\in \text{output-name} \\\\ +i, p &\in \text{deriving-path} \\\\ +d &\in \text{derivation} +\\end{align} + +\\begin{align} +\text{deriving-path} \quad p &::= s \mid (p, o) +\\end{align} diff --git a/doc/manual/source/store/resolution.md b/doc/manual/source/store/resolution.md new file mode 100644 index 000000000..9a87fea99 --- /dev/null +++ b/doc/manual/source/store/resolution.md @@ -0,0 +1,219 @@ +# Derivation Resolution + +*See [this chapter's appendix](@docroot@/store/math-notation.md) on grammar and metavariable conventions.* + +To *resolve* a derivation is to replace its [inputs] with the simplest inputs — plain store paths — that denote the same store objects. + +Derivations that only have store paths as inputs are likewise called *resolved derivations*. +(They are called that whether they are in fact the output of derivation resolution, or just made that way without non-store-path inputs to begin with.) + +## Input Content Equivalence of Derivations + +[Deriving paths][deriving-path] intentionally make it possible to refer to the same [store object] in multiple ways. +This is a consequence of content-addressing, since different derivations can produce the same outputs, and the same data can also be manually added to the store. +This is also a consequence even of input-addressing, as an output can be referred to by derivation and output name, or directly by its [computed](./derivation/outputs/input-address.md) store path. +Since dereferencing deriving paths is thus not injective, it induces an equivalence relation on deriving paths. + +Let's call this equivalence relation \\(\\sim\\), where \\(p_1 \\sim p_2\\) means that deriving paths \\(p_1\\) and \\(p_2\\) refer to the same store object. + +**Content Equivalence**: Two deriving paths are equivalent if they refer to the same store object: + +\\[ +\\begin{prooftree} +\\AxiomC{${}^*p_1 = {}^*p_2$} +\\UnaryInfC{$p_1 \\,\\sim_\\mathrm{DP}\\, p_2$} +\\end{prooftree} +\\] + +where \\({}^\*p\\) denotes the store object that deriving path \\(p\\) refers to. + +This also induces an equivalence relation on sets of deriving paths: + +\\[ +\\begin{prooftree} +\\AxiomC{$\\{ {}^*p | p \\in P_1 \\} = \\{ {}^*p | p \\in P_2 \\}$} +\\UnaryInfC{$P_1 \\,\\sim_{\\mathcal{P}(\\mathrm{DP})}\\, P_2$} +\\end{prooftree} +\\] + +**Input Content Equivalence**: This, in turn, induces an equivalence relation on derivations: two derivations are equivalent if their inputs are equivalent, and they are otherwise equal: + +\\[ +\\begin{prooftree} +\\AxiomC{$\\mathrm{inputs}(d_1) \\,\\sim_{\\mathcal{P}(\\mathrm{DP})}\\, \\mathrm{inputs}(d_2)$} +\\AxiomC{$ + d\_1\left[\\mathrm{inputs} := \\{\\}\right] + \= + d\_2\left[\\mathrm{inputs} := \\{\\}\right] +$} +\\BinaryInfC{$d_1 \\,\\sim_\\mathrm{Drv}\\, d_2$} +\\end{prooftree} +\\] + +Derivation resolution always maps derivations to input-content-equivalent derivations. + +## Resolution relation + +Dereferencing a derived path — \\({}^\*p\\) above — was just introduced as a black box. +But actually it is a multi-step process of looking up build results in the [build trace] that itself depends on resolving the lookup keys. +Resolution is thus a recursive multi-step process that is worth diagramming formally. + +We can do this with a small-step binary transition relation; let's call it \\(\rightsquigarrow\\). +We can then conclude dereferenced equality like this: + +\\[ +\\begin{prooftree} +\\AxiomC{$p\_1 \\rightsquigarrow^* p$} +\\AxiomC{$p\_2 \\rightsquigarrow^* p$} +\\BinaryInfC{${}^*p\_1 = {}^*p\_2$} +\\end{prooftree} +\\] + +I.e. by showing that both original items resolve (over 0 or more small steps, hence the \\({}^*\\)) to the same exact item. + +With this motivation, let's now formalize a [small-step](https://en.wikipedia.org/wiki/Operational_semantics#Small-step_semantics) system of reduction rules for resolution. + +### Formal rules + +### \\(\text{resolved}\\) unary relation + +\\[ +\\begin{prooftree} +\\AxiomC{$s \in \text{store-path}$} +\\UnaryInfC{$s$ resolved} +\\end{prooftree} +\\] + +\\[ +\\begin{prooftree} +\\AxiomC{$\forall i \in \mathrm{inputs}(d). i \text{ resolved}$} +\\UnaryInfC{$d$ resolved} +\\end{prooftree} +\\] + +### \\(\rightsquigarrow\\) binary relation + +> **Remark** +> +> Actually, to be completely formal we would need to keep track of the build trace we are choosing to resolve against. +> +> We could do that by making \\(\rightsquigarrow\\) a ternary relation, which would pass the build trace to itself until it finally uses it in that one rule. +> This would add clutter more than insight, so we didn't bother to write it. +> +> There are other options too, like saying the whole reduction rule system is parameterized on the build trace, essentially [currying](https://en.wikipedia.org/wiki/Currying) the ternary \\(\rightsquigarrow\\) into a function from build traces to the binary relation written above. + +#### Core build trace lookup rule + +\\[ +\\begin{prooftree} +\\AxiomC{$s \in \text{store-path}$} +\\AxiomC{${}^*s \in \text{derivation}$} +\\AxiomC{${}^*s$ resolved} +\\AxiomC{$\mathrm{build\text{-}trace}[s][o] = t$} +\\QuaternaryInfC{$(s, o) \rightsquigarrow t$} +\\RightLabel{\\scriptsize output path resolution} +\\end{prooftree} +\\] + +#### Inductive rules + +\\[ +\\begin{prooftree} +\\AxiomC{$i \\rightsquigarrow i'$} +\\AxiomC{$i \\in \\mathrm{inputs}(d)$} +\\BinaryInfC{$d \\rightsquigarrow d[i \\mapsto i']$} +\\end{prooftree} +\\] + +\\[ +\\begin{prooftree} +\\AxiomC{$d \\rightsquigarrow d'$} +\\UnaryInfC{$(\\mathrm{path}(d), o) \\rightsquigarrow (\\mathrm{path}(d'), o)$} +\\end{prooftree} +\\] + +\\[ +\\begin{prooftree} +\\AxiomC{$p \\rightsquigarrow p'$} +\\UnaryInfC{$(p, o) \\rightsquigarrow (p', o)$} +\\end{prooftree} +\\] + +### Properties + +Like all well-behaved evaluation relations, partial resolution is [*confluent*](https://en.wikipedia.org/wiki/Confluence_(abstract_rewriting)). +Also, if we take the symmetric closure of \\(\\rightsquigarrow^\*\\), we end up with the equivalence relations of the previous section. +Resolution respects content equivalence for deriving paths, and input content equivalence for derivations. + +> **Remark** +> +> We chose to define from scratch an "resolved" unary relation explicitly above. +> But it can also be defined as the normal forms of the \\(\\rightsquigarrow^\*\\) relation: +> +> \\[ a \text{ resolved} \Leftrightarrow \forall b. b \rightsquigarrow^* a \Rightarrow b = a\\] +> +> In prose, resolved terms are terms which \\(\\rightsquigarrow^\*\\) only relates on the left side to the same term on the right side; they are the terms which can be resolved no further. + +## Partial versus Complete Resolution + +Similar to evaluation, we can also speak of *partial* versus *complete* derivation resolution. +Partial derivation resolution is what we've actually formalized above with \\(\\rightsquigarrow^\*\\). +Complete resolution is resolution ending in a resolved term (deriving path or derivation). +(Which is a normal form of the relation, per the remark above.) + +With partial resolution, a derivation is related to equivalent derivations with the same or simpler inputs, but not all those inputs will be plain store paths. +This is useful when the input refers to a floating content addressed output we have not yet built — we don't know what (content-address) store path will used for that derivation, so we are "stuck" trying to resolve the deriving path in question. +(In the above formalization, this happens when the build trace is missing the keys we wish to look up in it.) + +Complete resolution is a *functional* relation, i.e. values on the left are uniquely related with values on the right. +It is not however, a *total* relation (in general, assuming arbitrary build traces). +This is discussed in the next section. + +## Termination + +For static derivations graphs, complete resolution is indeed total, because it always terminates for all inputs. +(A relation that is both total and functional is a function.) + +For [dynamic][xp-feature-dynamic-derivations] derivation graphs, however, this is not the case — resolution is not guaranteed to terminate. +The issue isn't rewriting deriving paths themselves: +a single rewrite to normalize an output deriving path to a constant one always exists, and always proceeds in one step. +The issue is that dynamic derivations (i.e. those that are filled-in the graph by a previous resolution) may have more transitive dependencies than the original derivation. + +> **Example** +> +> Suppose we have this deriving path +> ```json +> { +> "drvPath": { +> "drvPath": "...-foo.drv", +> "output": "bar.drv" +> }, +> "output": "baz" +> } +> ``` +> and derivation `foo` is already resolved. +> When we resolve deriving path we'll end up with something like. +> ```json +> { +> "drvPath": "...-foo-bar.drv", +> "output": "baz" +> } +> ``` +> So far is just an atomic single rewrite, with no termination issues. +> But the derivation `foo-bar` may have its *own* dynamic derivation inputs. +> Resolution must resolve that derivation first before the above deriving path can finally be normalized to a plain `...-foo-bar-baz` store path. + +The important thing to notice is that while "build trace" *keys* must be resolved. +The *value* those keys are mapped to have no such constraints. +An arbitrary store object has no notion of being resolved or not. +But, an arbitrary store object can be read back as a derivation (as will in fact be done in case for dynamic derivations / nested output deriving paths). +And those derivations need *not* be resolved. + +It is those dynamic non-resolved derivations which are the source of non-termination. +By the same token, they are also the reason why dynamic derivations offer greater expressive power. + +[store object]: @docroot@/store/store-object.md +[inputs]: @docroot@/store/derivation/index.md#inputs +[build trace]: @docroot@/store/build-trace.md +[deriving-path]: @docroot@/store/derivation/index.md#deriving-path +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs new file mode 100644 index 000000000..e514a9977 --- /dev/null +++ b/doc/manual/theme/head.hbs @@ -0,0 +1,15 @@ + + + diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 4615d8acd..259314d3f 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -277,7 +277,7 @@ struct BasicDerivation Path builder; Strings args; /** - * Must not contain the key `__json`, at least in order to serialize to A-Term. + * Must not contain the key `__json`, at least in order to serialize to ATerm. */ StringPairs env; std::optional structuredAttrs; From c3d4c5f69d93278dafe2c631f955ffe0e47ca689 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 22:00:54 +0000 Subject: [PATCH 178/201] build(deps): bump cachix/install-nix-action from 31.5.1 to 31.8.2 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.5.1 to 31.8.2. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/c134e4c9e34bac6cab09cf239815f9339aaaf84e...456688f15bc354bef6d396e4a35f4f89d40bf2b7) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.8.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..60c617978 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 + - uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31.8.2 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} From c8e24491c0da15406b736d11f5095a4c504d263c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 3 Nov 2025 14:19:54 -0800 Subject: [PATCH 179/201] Fix warning in kaitai spec Warning: ``` [39/483] Generating src/kaitai-struct-checks/kaitai-generated-sources with a custom command ../src/kaitai-struct-checks/nar.ksy: /types/padded_str/seq/1/encoding: warning: use canonical encoding name `ASCII` instead of `ascii` (see https://doc.kaitai.io/ksy_style_guide.html#encoding-name) ``` --- doc/manual/source/protocols/nix-archive/nar.ksy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/protocols/nix-archive/nar.ksy b/doc/manual/source/protocols/nix-archive/nar.ksy index 1cad09097..6a172b276 100644 --- a/doc/manual/source/protocols/nix-archive/nar.ksy +++ b/doc/manual/source/protocols/nix-archive/nar.ksy @@ -29,7 +29,7 @@ types: - id: body type: str size: len_str - encoding: 'ascii' + encoding: 'ASCII' - id: padding size: (8 - (len_str % 8)) % 8 From 469123eda19028ea784b78b6d21ae0d4e2c91ab3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 14:10:13 +0200 Subject: [PATCH 180/201] doc: Check link fragments with lychee --- flake.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a70617b74..897889a71 100644 --- a/flake.nix +++ b/flake.nix @@ -320,7 +320,16 @@ checks = forAllSystems ( system: - (import ./ci/gha/tests { + let + pkgs = nixpkgsFor.${system}.native; + in + { + # https://nixos.org/manual/nixpkgs/stable/index.html#tester-lycheeLinkCheck + linkcheck = pkgs.testers.lycheeLinkCheck { + site = self.packages.${system}.nix-manual + "/share/doc/nix/manual"; + }; + } + // (import ./ci/gha/tests { inherit system; pkgs = nixpkgsFor.${system}.native; nixFlake = self; From ae15d4eaf395f02a6b08e75044d5b5d48e1cb12c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 4 Nov 2025 00:18:51 +0100 Subject: [PATCH 181/201] Fix links in the manual --- doc/manual/anchors.jq | 2 +- doc/manual/source/command-ref/nix-channel.md | 2 +- .../source/command-ref/nix-env/upgrade.md | 2 +- doc/manual/source/development/building.md | 4 ++-- doc/manual/source/development/testing.md | 2 +- doc/manual/source/glossary.md | 10 +++++----- .../source/language/advanced-attributes.md | 2 +- doc/manual/source/language/derivations.md | 2 +- doc/manual/source/language/identifiers.md | 2 +- doc/manual/source/language/index.md | 10 +++++----- doc/manual/source/language/string-context.md | 2 +- .../source/language/string-interpolation.md | 2 +- doc/manual/source/language/syntax.md | 6 +++--- .../source/protocols/json/schema/hash-v1.yaml | 2 +- .../source/protocols/nix-archive/index.md | 2 +- doc/manual/source/release-notes/rl-2.18.md | 2 +- doc/manual/source/release-notes/rl-2.19.md | 4 ++-- doc/manual/source/release-notes/rl-2.23.md | 2 +- doc/manual/source/release-notes/rl-2.24.md | 2 +- doc/manual/source/store/building.md | 2 +- doc/manual/source/store/derivation/index.md | 2 +- .../source/store/derivation/outputs/index.md | 2 +- .../store/store-object/content-address.md | 4 ++-- doc/manual/source/store/store-path.md | 2 +- src/libexpr/primops.cc | 20 +++++++++---------- src/libexpr/primops/context.cc | 4 ++-- src/libstore/include/nix/store/globals.hh | 2 +- src/libstore/include/nix/store/local-store.hh | 2 +- src/libutil/experimental-features.cc | 2 +- src/nix/flake.md | 2 +- 30 files changed, 53 insertions(+), 53 deletions(-) diff --git a/doc/manual/anchors.jq b/doc/manual/anchors.jq index 72309779c..4ee2bc130 100755 --- a/doc/manual/anchors.jq +++ b/doc/manual/anchors.jq @@ -3,7 +3,7 @@ def transform_anchors_html: - . | gsub($empty_anchor_regex; "") + . | gsub($empty_anchor_regex; "") | gsub($anchor_regex; "" + .text + ""); diff --git a/doc/manual/source/command-ref/nix-channel.md b/doc/manual/source/command-ref/nix-channel.md index ed9cbb41f..3d02a7d40 100644 --- a/doc/manual/source/command-ref/nix-channel.md +++ b/doc/manual/source/command-ref/nix-channel.md @@ -14,7 +14,7 @@ The moving parts of channels are: - The official channels listed at - The user-specific list of [subscribed channels](#subscribed-channels) - The [downloaded channel contents](#channels) -- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-I) or the [`NIX_PATH` environment variable](#env-NIX_PATH) > **Note** > diff --git a/doc/manual/source/command-ref/nix-env/upgrade.md b/doc/manual/source/command-ref/nix-env/upgrade.md index 2779363c3..bf4c1a8ed 100644 --- a/doc/manual/source/command-ref/nix-env/upgrade.md +++ b/doc/manual/source/command-ref/nix-env/upgrade.md @@ -22,7 +22,7 @@ left untouched; this is not an error. It is also not an error if an element of *args* matches no installed derivations. For a description of how *args* is mapped to a set of store paths, see -[`--install`](#operation---install). If *args* describes multiple +[`--install`](./install.md). If *args* describes multiple store paths with the same symbolic name, only the one with the highest version is installed. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 889d81d80..eb65a7247 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -66,7 +66,7 @@ You can also build Nix for one of the [supported platforms](#platforms). This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. [`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command +[`nix-command`]: @docroot@/development/experimental-features.md#xp-feature-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -256,7 +256,7 @@ You can use any of the other supported environments in place of `nix-cli-ccacheS ## Editor integration The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. -See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). +See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#building-nix-with-flakes) or in [classic Nix](#building-nix). To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. Meson's configure always produces this inside the build directory. diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index c0b130155..7c2cbbb5d 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -119,7 +119,7 @@ This will: 3. Stop the program when the test fails, allowing the user to then issue arbitrary commands to GDB. -### Characterisation testing { #characaterisation-testing-unit } +### Characterisation testing { #characterisation-testing-unit } See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index e6a294e7d..502e6d4de 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -208,7 +208,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -279,7 +279,7 @@ See [References](@docroot@/store/store-object.md#references) for details. -- [referrer]{#gloss-reference} +- [referrer]{#gloss-referrer} A reversed edge from one [store object] to another. @@ -367,8 +367,8 @@ Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. - The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#attribute-set) containing: - - attributes that refer to the files of a package, typically in the form of [derivation outputs](#output), + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), - attributes with metadata, such as information about how the package is supposed to be used. The exact shape of these attribute sets is up to convention. @@ -383,7 +383,7 @@ [string]: ./language/types.md#type-string [path]: ./language/types.md#type-path - [attribute name]: ./language/types.md#attribute-set + [attribute name]: ./language/types.md#type-attrs - [base directory]{#gloss-base-directory} diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index c9d64f060..f0b1a4c73 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -333,7 +333,7 @@ Here is more information on the `output*` attributes, and what values they may b `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format, because in that case the choice of hash algorithm is determined by `outputHash`. - - [`outputHash`]{#adv-attr-outputHashAlgo}; [`outputHash`]{#adv-attr-outputHashMode} + - [`outputHash`]{#adv-attr-outputHash} This will specify the output hash of the single output of a [fixed-output derivation]. diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 43eec680b..2403183fc 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -16,7 +16,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. - See [derivation outputs](@docroot@/store/derivation/index.md#outputs) for what this is affects. + See [derivation outputs](@docroot@/store/derivation/outputs/index.md#outputs) for what this is affects. [store path]: @docroot@/store/store-path.md diff --git a/doc/manual/source/language/identifiers.md b/doc/manual/source/language/identifiers.md index 584a2f861..67bb1eeec 100644 --- a/doc/manual/source/language/identifiers.md +++ b/doc/manual/source/language/identifiers.md @@ -16,7 +16,7 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A *name* can be written as an [identifier](#identifier) or a [string literal](./string-literals.md). +A *name* can be written as an [identifier](#identifiers) or a [string literal](./string-literals.md). > **Syntax** > diff --git a/doc/manual/source/language/index.md b/doc/manual/source/language/index.md index 1eb14e96d..116f928dc 100644 --- a/doc/manual/source/language/index.md +++ b/doc/manual/source/language/index.md @@ -137,7 +137,7 @@ This is an incomplete overview of language features, by example. - [Booleans](@docroot@/language/types.md#type-boolean) + [Booleans](@docroot@/language/types.md#type-bool) @@ -245,7 +245,7 @@ This is an incomplete overview of language features, by example. - An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#type-attrs) with attributes named `x` and `y` @@ -285,7 +285,7 @@ This is an incomplete overview of language features, by example. - [Lists](@docroot@/language/types.md#list) with three elements. + [Lists](@docroot@/language/types.md#type-list) with three elements. @@ -369,7 +369,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#type-attrs) (evaluates to `1`) @@ -381,7 +381,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#type-attrs) with default (evaluates to `3`) diff --git a/doc/manual/source/language/string-context.md b/doc/manual/source/language/string-context.md index 0d8fcdefa..65c59d865 100644 --- a/doc/manual/source/language/string-context.md +++ b/doc/manual/source/language/string-context.md @@ -111,7 +111,7 @@ It creates an [attribute set] representing the string context, which can be insp [`builtins.hasContext`]: ./builtins.md#builtins-hasContext [`builtins.getContext`]: ./builtins.md#builtins-getContext -[attribute set]: ./types.md#attribute-set +[attribute set]: ./types.md#type-attrs ## Clearing string contexts diff --git a/doc/manual/source/language/string-interpolation.md b/doc/manual/source/language/string-interpolation.md index a503d5f04..8e25d2b63 100644 --- a/doc/manual/source/language/string-interpolation.md +++ b/doc/manual/source/language/string-interpolation.md @@ -6,7 +6,7 @@ Such a construct is called *interpolated string*, and the expression inside is a [string]: ./types.md#type-string [path]: ./types.md#type-path -[attribute set]: ./types.md#attribute-set +[attribute set]: ./types.md#type-attrs > **Syntax** > diff --git a/doc/manual/source/language/syntax.md b/doc/manual/source/language/syntax.md index 85162db74..b127aca14 100644 --- a/doc/manual/source/language/syntax.md +++ b/doc/manual/source/language/syntax.md @@ -51,7 +51,7 @@ See [String literals](string-literals.md). Path literals can also include [string interpolation], besides being [interpolated into other expressions]. - [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expression At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. @@ -235,7 +235,7 @@ of object-oriented programming, for example. ## Recursive sets -Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. +Recursive sets are like normal [attribute sets](./types.md#type-attrs), but the attributes can refer to each other. > *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` @@ -287,7 +287,7 @@ This evaluates to `"foobar"`. ## Inheriting attributes -When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +When defining an [attribute set](./types.md#type-attrs) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). This can be shortened using the `inherit` keyword. Example: diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml index 316fb6d73..821546dee 100644 --- a/doc/manual/source/protocols/json/schema/hash-v1.yaml +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -51,4 +51,4 @@ additionalProperties: false description: | The hash algorithm used to compute the hash value. - `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake-hashing) experimental feature. + `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake3-hashes) experimental feature. diff --git a/doc/manual/source/protocols/nix-archive/index.md b/doc/manual/source/protocols/nix-archive/index.md index 4d25f63e2..bd2a8e833 100644 --- a/doc/manual/source/protocols/nix-archive/index.md +++ b/doc/manual/source/protocols/nix-archive/index.md @@ -4,7 +4,7 @@ This is the complete specification of the [Nix Archive] format. The Nix Archive format closely follows the abstract specification of a [file system object] tree, because it is designed to serialize exactly that data structure. -[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive [file system object]: @docroot@/store/file-system-object.md The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. diff --git a/doc/manual/source/release-notes/rl-2.18.md b/doc/manual/source/release-notes/rl-2.18.md index eb26fc9e7..71b25f408 100644 --- a/doc/manual/source/release-notes/rl-2.18.md +++ b/doc/manual/source/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/language/advanced-attributes.md#adv-attr-unsafeDiscardReferences) attribute is no longer guarded by an experimental flag and can be used freely. diff --git a/doc/manual/source/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md index 06c704324..04f8c9c28 100644 --- a/doc/manual/source/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-feature-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes). - The interface for creating and updating lock files has been overhauled: diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index e6b0e9ffc..b358a0fdc 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. diff --git a/doc/manual/source/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md index d4af3cb51..e9b46bb22 100644 --- a/doc/manual/source/release-notes/rl-2.24.md +++ b/doc/manual/source/release-notes/rl-2.24.md @@ -93,7 +93,7 @@ - Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) - Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/conf-file.md#conf-min-free) to set the minimum free space to 1 gigabyte. This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index dbfe6b5ca..f2d470e99 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -8,7 +8,7 @@ - Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. -## Builder Execution +## Builder Execution {#builder-execution} The [`builder`](./derivation/index.md#builder) is executed as follows: diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 61c5335ff..670f3b2bd 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -102,7 +102,7 @@ But rather than somehow scanning all the other fields for inputs, Nix requires t ### System {#system} -The system type on which the [`builder`](#attr-builder) executable is meant to be run. +The system type on which the [`builder`](#builder) executable is meant to be run. A necessary condition for Nix to schedule a given derivation on some [Nix instance] is for the "system" of that derivation to match that instance's [`system` configuration option] or [`extra-platforms` configuration option]. diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index 0683f5703..ca2ce6665 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -43,7 +43,7 @@ In particular, the specification decides: - if the content is content-addressed, how is it content addressed -- if the content is content-addressed, [what is its content address](./content-address.md#fixed-content-addressing) (and thus what is its [store path]) +- if the content is content-addressed, [what is its content address](./content-address.md#fixed) (and thus what is its [store path]) ## Types of derivations diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 36e841fa3..7834ac510 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -1,7 +1,7 @@ # Content-Addressing Store Objects Just [like][fso-ca] [File System Objects][File System Object], -[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-address), unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. @@ -107,7 +107,7 @@ References (to other store objects and self-references alike) are supported so l > > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. -This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. +This uses the corresponding [Git](../file-system-object/content-address.md#git) method of file system object content addressing. References are not supported. diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index beec2389b..4061f3653 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -6,7 +6,7 @@ > > A rendered store path -Nix implements references to [store objects](./index.md#store-object) as *store paths*. +Nix implements references to [store objects](./store-object.md) as *store paths*. Think of a store path as an [opaque], [unique identifier]: The only way to obtain store path is by adding or building store objects. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 96e79fedd..d1aae64fa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -825,10 +825,10 @@ static RegisterPrimOp primop_genericClosure( - [Int](@docroot@/language/types.md#type-int) - [Float](@docroot@/language/types.md#type-float) - - [Boolean](@docroot@/language/types.md#type-boolean) + - [Boolean](@docroot@/language/types.md#type-bool) - [String](@docroot@/language/types.md#type-string) - [Path](@docroot@/language/types.md#type-path) - - [List](@docroot@/language/types.md#list) + - [List](@docroot@/language/types.md#type-list) The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added. Items are compared by their `key` attribute. @@ -2103,7 +2103,7 @@ static RegisterPrimOp primop_findFile( builtins.findFile builtins.nixPath "nixpkgs" ``` - A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + A search path is represented as a list of [attribute sets](./types.md#type-attrs) with two attributes: - `prefix` is a relative path. - `path` denotes a file system location @@ -2395,7 +2395,7 @@ static RegisterPrimOp primop_outputOf({ returns an input placeholder for the output of the output of `myDrv`. - This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-paths), e.g. as part of installable syntax on the command line. + This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-path), e.g. as part of installable syntax on the command line. )", .fun = prim_outputOf, .experimentalFeature = Xp::DynamicDerivations, @@ -4966,7 +4966,7 @@ static RegisterPrimOp primop_compareVersions({ version *s1* is older than version *s2*, `0` if they are the same, and `1` if *s1* is newer than *s2*. The version comparison algorithm is the same as the one used by [`nix-env - -u`](../command-ref/nix-env.md#operation---upgrade). + -u`](../command-ref/nix-env/upgrade.md). )", .fun = prim_compareVersions, }); @@ -4995,7 +4995,7 @@ static RegisterPrimOp primop_splitVersion({ .doc = R"( Split a string representing a version into its components, by the same version splitting logic underlying the version comparison in - [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade). + [`nix-env -u`](../command-ref/nix-env/upgrade.md). )", .fun = prim_splitVersion, }); @@ -5045,9 +5045,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) Primitive value. It can be returned by - [comparison operators](@docroot@/language/operators.md#Comparison) + [comparison operators](@docroot@/language/operators.md#comparison) and used in - [conditional expressions](@docroot@/language/syntax.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#conditionals). The name `true` is not special, and can be shadowed: @@ -5068,9 +5068,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) Primitive value. It can be returned by - [comparison operators](@docroot@/language/operators.md#Comparison) + [comparison operators](@docroot@/language/operators.md#comparison) and used in - [conditional expressions](@docroot@/language/syntax.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#conditionals). The name `false` is not special, and can be shadowed: diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 12b8ffdf9..8a9fe42e8 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -79,7 +79,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency( Create a copy of the given string where every [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) string context element is turned into a - [constant](@docroot@/language/string-context.md#string-context-element-constant) + [constant](@docroot@/language/string-context.md#string-context-constant) string context element. This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). @@ -145,7 +145,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies( .args = {"s"}, .doc = R"( Create a copy of the given string where a single - [constant](@docroot@/language/string-context.md#string-context-element-constant) + [constant](@docroot@/language/string-context.md#string-context-constant) string context element is turned into a [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) string context element. diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 8aa82c4a2..5ddfbee30 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -189,7 +189,7 @@ public: 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism.