1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-08 19:46:02 +01:00

refactor(libstore): minimize NIX_WITH_S3_SUPPORT scope to auth only

Move S3 URL parsing, store configuration, and public bucket support
outside of NIX_WITH_S3_SUPPORT guards. Only AWS credential resolution
remains gated, allowing builds with withAWS = false to:

- Parse s3:// URLs
- Register S3 store types
- Access public S3 buckets (via HTTPS conversion)
- Use S3-compatible services without authentication

The setupForS3() function now always performs URL conversion, with
authentication code conditionally compiled based on NIX_WITH_S3_SUPPORT.
The aws-creds.cc file (only code using AWS CRT SDK) is now conditionally
compiled by meson.
This commit is contained in:
Bernardo Meurer Costa 2025-10-15 18:07:42 +00:00
parent 1f710300c9
commit bb1f22a8df
No known key found for this signature in database
9 changed files with 40 additions and 67 deletions

View file

@ -1,12 +1,9 @@
#include "nix/store/s3-binary-cache-store.hh" #include "nix/store/s3-binary-cache-store.hh"
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/filetransfer.hh"
#include "nix/store/s3-url.hh"
#if NIX_WITH_S3_SUPPORT #include <gtest/gtest.h>
# include "nix/store/http-binary-cache-store.hh"
# include "nix/store/filetransfer.hh"
# include "nix/store/s3-url.hh"
# include <gtest/gtest.h>
namespace nix { namespace nix {
@ -126,5 +123,3 @@ TEST(S3BinaryCacheStore, parameterFiltering)
} }
} // namespace nix } // namespace nix
#endif

View file

@ -1,10 +1,8 @@
#include "nix/store/s3-url.hh" #include "nix/store/s3-url.hh"
#include "nix/util/tests/gmock-matchers.hh" #include "nix/util/tests/gmock-matchers.hh"
#if NIX_WITH_S3_SUPPORT #include <gtest/gtest.h>
#include <gmock/gmock.h>
# include <gtest/gtest.h>
# include <gmock/gmock.h>
namespace nix { namespace nix {
@ -228,5 +226,3 @@ INSTANTIATE_TEST_SUITE_P(
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; }); [](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
} // namespace nix } // namespace nix
#endif

View file

@ -8,10 +8,10 @@
#include "nix/util/signals.hh" #include "nix/util/signals.hh"
#include "store-config-private.hh" #include "store-config-private.hh"
#include "nix/store/s3-url.hh"
#include <optional> #include <optional>
#if NIX_WITH_S3_SUPPORT #if NIX_WITH_S3_SUPPORT
# include "nix/store/aws-creds.hh" # include "nix/store/aws-creds.hh"
# include "nix/store/s3-url.hh"
#endif #endif
#ifdef __linux__ #ifdef __linux__
@ -820,10 +820,7 @@ struct curlFileTransfer : public FileTransfer
void enqueueItem(std::shared_ptr<TransferItem> item) void enqueueItem(std::shared_ptr<TransferItem> item)
{ {
if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
#if NIX_WITH_S3_SUPPORT && item->request.uri.scheme() != "s3")
&& item->request.uri.scheme() != "s3"
#endif
)
throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string());
{ {
@ -839,16 +836,11 @@ struct curlFileTransfer : public FileTransfer
void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
{ {
/* Ugly hack to support s3:// URIs. */ /* Handle s3:// URIs by converting to HTTPS and optionally adding auth */
if (request.uri.scheme() == "s3") { if (request.uri.scheme() == "s3") {
#if NIX_WITH_S3_SUPPORT
// New curl-based S3 implementation
auto modifiedRequest = request; auto modifiedRequest = request;
modifiedRequest.setupForS3(); modifiedRequest.setupForS3();
enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback))); enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
#else
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri.to_string());
#endif
return; return;
} }
@ -876,14 +868,16 @@ ref<FileTransfer> makeFileTransfer()
return makeCurlFileTransfer(); return makeCurlFileTransfer();
} }
#if NIX_WITH_S3_SUPPORT
void FileTransferRequest::setupForS3() void FileTransferRequest::setupForS3()
{ {
auto parsedS3 = ParsedS3URL::parse(uri.parsed()); auto parsedS3 = ParsedS3URL::parse(uri.parsed());
// Update the request URI to use HTTPS // Update the request URI to use HTTPS (works without AWS SDK)
uri = parsedS3.toHttpsUrl(); uri = parsedS3.toHttpsUrl();
// This gets used later in a curl setopt
#if NIX_WITH_S3_SUPPORT
// Auth-specific code only compiled when AWS support is available
awsSigV4Provider = "aws:amz:" + parsedS3.region.value_or("us-east-1") + ":s3"; awsSigV4Provider = "aws:amz:" + parsedS3.region.value_or("us-east-1") + ":s3";
// check if the request already has pre-resolved credentials // check if the request already has pre-resolved credentials
std::optional<std::string> sessionToken; std::optional<std::string> sessionToken;
if (usernameAuth) { if (usernameAuth) {
@ -908,8 +902,11 @@ void FileTransferRequest::setupForS3()
} }
if (sessionToken) if (sessionToken)
headers.emplace_back("x-amz-security-token", *sessionToken); headers.emplace_back("x-amz-security-token", *sessionToken);
} #else
// When built without AWS support, just try as public bucket
debug("S3 request without authentication (built without AWS support)");
#endif #endif
}
std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
{ {

View file

@ -15,6 +15,7 @@
#if NIX_WITH_S3_SUPPORT #if NIX_WITH_S3_SUPPORT
# include "nix/store/aws-creds.hh" # include "nix/store/aws-creds.hh"
#endif #endif
#include "nix/store/s3-url.hh"
namespace nix { namespace nix {
@ -132,10 +133,10 @@ struct FileTransferRequest
return data ? "upload" : "download"; return data ? "upload" : "download";
} }
#if NIX_WITH_S3_SUPPORT
private: private:
friend struct curlFileTransfer; friend struct curlFileTransfer;
void setupForS3(); void setupForS3();
#if NIX_WITH_S3_SUPPORT
std::optional<std::string> awsSigV4Provider; std::optional<std::string> awsSigV4Provider;
#endif #endif
}; };

View file

@ -2,10 +2,7 @@
///@file ///@file
#include "nix/store/config.hh" #include "nix/store/config.hh"
#include "nix/store/http-binary-cache-store.hh"
#if NIX_WITH_S3_SUPPORT
# include "nix/store/http-binary-cache-store.hh"
namespace nix { namespace nix {
@ -77,5 +74,3 @@ public:
}; };
} // namespace nix } // namespace nix
#endif

View file

@ -1,16 +1,13 @@
#pragma once #pragma once
///@file ///@file
#include "nix/store/config.hh" #include "nix/store/config.hh"
#include "nix/util/url.hh"
#include "nix/util/util.hh"
#if NIX_WITH_S3_SUPPORT #include <optional>
#include <string>
# include "nix/util/url.hh" #include <variant>
# include "nix/util/util.hh" #include <vector>
# include <optional>
# include <string>
# include <variant>
# include <vector>
namespace nix { namespace nix {
@ -56,5 +53,3 @@ struct ParsedS3URL
}; };
} // namespace nix } // namespace nix
#endif

View file

@ -265,7 +265,6 @@ config_priv_h = configure_file(
subdir('nix-meson-build-support/common') subdir('nix-meson-build-support/common')
sources = files( sources = files(
'aws-creds.cc',
'binary-cache-store.cc', 'binary-cache-store.cc',
'build-result.cc', 'build-result.cc',
'build/derivation-builder.cc', 'build/derivation-builder.cc',
@ -344,6 +343,11 @@ sources = files(
'worker-protocol.cc', 'worker-protocol.cc',
) )
# AWS credentials code requires AWS CRT, so only compile when enabled
if curl_s3_store_opt.enabled()
sources += files('aws-creds.cc')
endif
subdir('include/nix/store') subdir('include/nix/store')
if host_machine.system() == 'linux' if host_machine.system() == 'linux'

View file

@ -1,12 +1,9 @@
#include "nix/store/s3-binary-cache-store.hh" #include "nix/store/s3-binary-cache-store.hh"
#if NIX_WITH_S3_SUPPORT #include <cassert>
# include <cassert> #include "nix/store/http-binary-cache-store.hh"
#include "nix/store/store-registration.hh"
# include "nix/store/s3-binary-cache-store.hh"
# include "nix/store/http-binary-cache-store.hh"
# include "nix/store/store-registration.hh"
namespace nix { namespace nix {
@ -45,5 +42,3 @@ std::string S3BinaryCacheStoreConfig::doc()
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore; static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
} // namespace nix } // namespace nix
#endif

View file

@ -1,13 +1,10 @@
#include "nix/store/s3-url.hh" #include "nix/store/s3-url.hh"
#include "nix/util/error.hh"
#include "nix/util/split.hh"
#include "nix/util/strings-inline.hh"
#if NIX_WITH_S3_SUPPORT #include <ranges>
#include <string_view>
# include "nix/util/error.hh"
# include "nix/util/split.hh"
# include "nix/util/strings-inline.hh"
# include <ranges>
# include <string_view>
using namespace std::string_view_literals; using namespace std::string_view_literals;
@ -117,5 +114,3 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
} }
} // namespace nix } // namespace nix
#endif