mirror of
https://github.com/NixOS/nix.git
synced 2025-11-09 03:56:01 +01:00
refactor(libstore): minimize NIX_WITH_S3_SUPPORT scope to auth only
Move S3 URL parsing, store configuration, and public bucket support outside of NIX_WITH_S3_SUPPORT guards. Only AWS credential resolution remains gated, allowing builds with withAWS = false to: - Parse s3:// URLs - Register S3 store types - Access public S3 buckets (via HTTPS conversion) - Use S3-compatible services without authentication The setupForS3() function now always performs URL conversion, with authentication code conditionally compiled based on NIX_WITH_S3_SUPPORT. The aws-creds.cc file (only code using AWS CRT SDK) is now conditionally compiled by meson.
This commit is contained in:
parent
1f710300c9
commit
bb1f22a8df
9 changed files with 40 additions and 67 deletions
|
|
@ -1,7 +1,4 @@
|
||||||
#include "nix/store/s3-binary-cache-store.hh"
|
#include "nix/store/s3-binary-cache-store.hh"
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
|
|
||||||
#include "nix/store/http-binary-cache-store.hh"
|
#include "nix/store/http-binary-cache-store.hh"
|
||||||
#include "nix/store/filetransfer.hh"
|
#include "nix/store/filetransfer.hh"
|
||||||
#include "nix/store/s3-url.hh"
|
#include "nix/store/s3-url.hh"
|
||||||
|
|
@ -126,5 +123,3 @@ TEST(S3BinaryCacheStore, parameterFiltering)
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,6 @@
|
||||||
#include "nix/store/s3-url.hh"
|
#include "nix/store/s3-url.hh"
|
||||||
#include "nix/util/tests/gmock-matchers.hh"
|
#include "nix/util/tests/gmock-matchers.hh"
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
|
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
#include <gmock/gmock.h>
|
#include <gmock/gmock.h>
|
||||||
|
|
||||||
|
|
@ -228,5 +226,3 @@ INSTANTIATE_TEST_SUITE_P(
|
||||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
|
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
|
||||||
|
|
@ -8,10 +8,10 @@
|
||||||
#include "nix/util/signals.hh"
|
#include "nix/util/signals.hh"
|
||||||
|
|
||||||
#include "store-config-private.hh"
|
#include "store-config-private.hh"
|
||||||
|
#include "nix/store/s3-url.hh"
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#if NIX_WITH_S3_SUPPORT
|
#if NIX_WITH_S3_SUPPORT
|
||||||
# include "nix/store/aws-creds.hh"
|
# include "nix/store/aws-creds.hh"
|
||||||
# include "nix/store/s3-url.hh"
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef __linux__
|
#ifdef __linux__
|
||||||
|
|
@ -820,10 +820,7 @@ struct curlFileTransfer : public FileTransfer
|
||||||
void enqueueItem(std::shared_ptr<TransferItem> item)
|
void enqueueItem(std::shared_ptr<TransferItem> item)
|
||||||
{
|
{
|
||||||
if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
|
if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
|
||||||
#if NIX_WITH_S3_SUPPORT
|
&& item->request.uri.scheme() != "s3")
|
||||||
&& item->request.uri.scheme() != "s3"
|
|
||||||
#endif
|
|
||||||
)
|
|
||||||
throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string());
|
throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string());
|
||||||
|
|
||||||
{
|
{
|
||||||
|
|
@ -839,16 +836,11 @@ struct curlFileTransfer : public FileTransfer
|
||||||
|
|
||||||
void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
|
void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
|
||||||
{
|
{
|
||||||
/* Ugly hack to support s3:// URIs. */
|
/* Handle s3:// URIs by converting to HTTPS and optionally adding auth */
|
||||||
if (request.uri.scheme() == "s3") {
|
if (request.uri.scheme() == "s3") {
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
// New curl-based S3 implementation
|
|
||||||
auto modifiedRequest = request;
|
auto modifiedRequest = request;
|
||||||
modifiedRequest.setupForS3();
|
modifiedRequest.setupForS3();
|
||||||
enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
|
enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
|
||||||
#else
|
|
||||||
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri.to_string());
|
|
||||||
#endif
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -876,14 +868,16 @@ ref<FileTransfer> makeFileTransfer()
|
||||||
return makeCurlFileTransfer();
|
return makeCurlFileTransfer();
|
||||||
}
|
}
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
void FileTransferRequest::setupForS3()
|
void FileTransferRequest::setupForS3()
|
||||||
{
|
{
|
||||||
auto parsedS3 = ParsedS3URL::parse(uri.parsed());
|
auto parsedS3 = ParsedS3URL::parse(uri.parsed());
|
||||||
// Update the request URI to use HTTPS
|
// Update the request URI to use HTTPS (works without AWS SDK)
|
||||||
uri = parsedS3.toHttpsUrl();
|
uri = parsedS3.toHttpsUrl();
|
||||||
// This gets used later in a curl setopt
|
|
||||||
|
#if NIX_WITH_S3_SUPPORT
|
||||||
|
// Auth-specific code only compiled when AWS support is available
|
||||||
awsSigV4Provider = "aws:amz:" + parsedS3.region.value_or("us-east-1") + ":s3";
|
awsSigV4Provider = "aws:amz:" + parsedS3.region.value_or("us-east-1") + ":s3";
|
||||||
|
|
||||||
// check if the request already has pre-resolved credentials
|
// check if the request already has pre-resolved credentials
|
||||||
std::optional<std::string> sessionToken;
|
std::optional<std::string> sessionToken;
|
||||||
if (usernameAuth) {
|
if (usernameAuth) {
|
||||||
|
|
@ -908,8 +902,11 @@ void FileTransferRequest::setupForS3()
|
||||||
}
|
}
|
||||||
if (sessionToken)
|
if (sessionToken)
|
||||||
headers.emplace_back("x-amz-security-token", *sessionToken);
|
headers.emplace_back("x-amz-security-token", *sessionToken);
|
||||||
}
|
#else
|
||||||
|
// When built without AWS support, just try as public bucket
|
||||||
|
debug("S3 request without authentication (built without AWS support)");
|
||||||
#endif
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
|
std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@
|
||||||
#if NIX_WITH_S3_SUPPORT
|
#if NIX_WITH_S3_SUPPORT
|
||||||
# include "nix/store/aws-creds.hh"
|
# include "nix/store/aws-creds.hh"
|
||||||
#endif
|
#endif
|
||||||
|
#include "nix/store/s3-url.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
@ -132,10 +133,10 @@ struct FileTransferRequest
|
||||||
return data ? "upload" : "download";
|
return data ? "upload" : "download";
|
||||||
}
|
}
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
private:
|
private:
|
||||||
friend struct curlFileTransfer;
|
friend struct curlFileTransfer;
|
||||||
void setupForS3();
|
void setupForS3();
|
||||||
|
#if NIX_WITH_S3_SUPPORT
|
||||||
std::optional<std::string> awsSigV4Provider;
|
std::optional<std::string> awsSigV4Provider;
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,6 @@
|
||||||
///@file
|
///@file
|
||||||
|
|
||||||
#include "nix/store/config.hh"
|
#include "nix/store/config.hh"
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
|
|
||||||
#include "nix/store/http-binary-cache-store.hh"
|
#include "nix/store/http-binary-cache-store.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
@ -77,5 +74,3 @@ public:
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,6 @@
|
||||||
#pragma once
|
#pragma once
|
||||||
///@file
|
///@file
|
||||||
#include "nix/store/config.hh"
|
#include "nix/store/config.hh"
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
|
|
||||||
#include "nix/util/url.hh"
|
#include "nix/util/url.hh"
|
||||||
#include "nix/util/util.hh"
|
#include "nix/util/util.hh"
|
||||||
|
|
||||||
|
|
@ -56,5 +53,3 @@ struct ParsedS3URL
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
|
||||||
|
|
@ -265,7 +265,6 @@ config_priv_h = configure_file(
|
||||||
subdir('nix-meson-build-support/common')
|
subdir('nix-meson-build-support/common')
|
||||||
|
|
||||||
sources = files(
|
sources = files(
|
||||||
'aws-creds.cc',
|
|
||||||
'binary-cache-store.cc',
|
'binary-cache-store.cc',
|
||||||
'build-result.cc',
|
'build-result.cc',
|
||||||
'build/derivation-builder.cc',
|
'build/derivation-builder.cc',
|
||||||
|
|
@ -344,6 +343,11 @@ sources = files(
|
||||||
'worker-protocol.cc',
|
'worker-protocol.cc',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# AWS credentials code requires AWS CRT, so only compile when enabled
|
||||||
|
if curl_s3_store_opt.enabled()
|
||||||
|
sources += files('aws-creds.cc')
|
||||||
|
endif
|
||||||
|
|
||||||
subdir('include/nix/store')
|
subdir('include/nix/store')
|
||||||
|
|
||||||
if host_machine.system() == 'linux'
|
if host_machine.system() == 'linux'
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,7 @@
|
||||||
#include "nix/store/s3-binary-cache-store.hh"
|
#include "nix/store/s3-binary-cache-store.hh"
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
|
|
||||||
#include <cassert>
|
#include <cassert>
|
||||||
|
|
||||||
# include "nix/store/s3-binary-cache-store.hh"
|
|
||||||
#include "nix/store/http-binary-cache-store.hh"
|
#include "nix/store/http-binary-cache-store.hh"
|
||||||
#include "nix/store/store-registration.hh"
|
#include "nix/store/store-registration.hh"
|
||||||
|
|
||||||
|
|
@ -45,5 +42,3 @@ std::string S3BinaryCacheStoreConfig::doc()
|
||||||
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
|
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,4 @@
|
||||||
#include "nix/store/s3-url.hh"
|
#include "nix/store/s3-url.hh"
|
||||||
|
|
||||||
#if NIX_WITH_S3_SUPPORT
|
|
||||||
|
|
||||||
#include "nix/util/error.hh"
|
#include "nix/util/error.hh"
|
||||||
#include "nix/util/split.hh"
|
#include "nix/util/split.hh"
|
||||||
#include "nix/util/strings-inline.hh"
|
#include "nix/util/strings-inline.hh"
|
||||||
|
|
@ -117,5 +114,3 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue