diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index dc8584e1b..cd404a554 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -24,6 +24,22 @@ namespace nix { namespace { +// Global credential provider cache using boost's concurrent map +// Key: profile name (empty string for default profile) +using CredentialProviderCache = + boost::concurrent_flat_map>; + +static CredentialProviderCache credentialProviderCache; + +/** + * Clear all cached credential providers. + * Called automatically by CrtWrapper destructor during static destruction. + */ +static void clearAwsCredentialsCache() +{ + credentialProviderCache.clear(); +} + static void initAwsCrt() { struct CrtWrapper @@ -95,13 +111,6 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr>; - -static CredentialProviderCache credentialProviderCache; - } // anonymous namespace AwsCredentials getAwsCredentials(const std::string & profile) @@ -160,11 +169,6 @@ void invalidateAwsCredentials(const std::string & profile) credentialProviderCache.erase(profile); } -void clearAwsCredentialsCache() -{ - credentialProviderCache.clear(); -} - AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url) { std::string profile = s3Url.profile.value_or(""); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 59fc75ed0..d6e21f3e6 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -9,9 +9,14 @@ #include "nix/util/signals.hh" #include "store-config-private.hh" +#include #if NIX_WITH_S3_SUPPORT # include #endif +#if NIX_WITH_CURL_S3 +# include "nix/store/aws-creds.hh" +# include "nix/store/s3-url.hh" +#endif #ifdef __linux__ # include "nix/util/linux-namespaces.hh" @@ -426,6 +431,24 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf); errbuf[0] = 0; + // Set up username/password authentication if provided + if (request.usernameAuth) { + curl_easy_setopt(req, CURLOPT_USERNAME, request.usernameAuth->username.c_str()); + if (request.usernameAuth->password) { + curl_easy_setopt(req, CURLOPT_PASSWORD, request.usernameAuth->password->c_str()); + } + } + +#if NIX_WITH_CURL_S3 + // Set up AWS SigV4 signing if this is an S3 request + // Note: AWS SigV4 support guaranteed available (curl >= 7.75.0 checked at build time) + // The username/password (access key ID and secret key) are set via the general + // usernameAuth mechanism above. + if (request.awsSigV4Provider) { + curl_easy_setopt(req, CURLOPT_AWS_SIGV4, request.awsSigV4Provider->c_str()); + } +#endif + result.data.clear(); result.bodySize = 0; } @@ -800,7 +823,11 @@ struct curlFileTransfer : public FileTransfer void enqueueItem(std::shared_ptr item) { - if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https") + if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" +#if NIX_WITH_CURL_S3 + && item->request.uri.scheme() != "s3" +#endif + ) throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); { @@ -818,9 +845,15 @@ struct curlFileTransfer : public FileTransfer { /* Ugly hack to support s3:// URIs. */ if (request.uri.scheme() == "s3") { +#if NIX_WITH_CURL_S3 + // New curl-based S3 implementation + auto modifiedRequest = request; + modifiedRequest.setupForS3(); + enqueueItem(std::make_shared(*this, std::move(modifiedRequest), std::move(callback))); +#elif NIX_WITH_S3_SUPPORT + // Old AWS SDK-based implementation // FIXME: do this on a worker thread try { -#if NIX_WITH_S3_SUPPORT auto parsed = ParsedS3URL::parse(request.uri.parsed()); std::string profile = parsed.profile.value_or(""); @@ -838,13 +871,12 @@ struct curlFileTransfer : public FileTransfer res.data = std::move(*s3Res.data); res.urls.push_back(request.uri.to_string()); callback(std::move(res)); -#else - throw nix::Error( - "cannot download '%s' because Nix is not built with S3 support", request.uri.to_string()); -#endif } catch (...) { callback.rethrow(); } +#else + throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri.to_string()); +#endif return; } @@ -872,6 +904,41 @@ ref makeFileTransfer() return makeCurlFileTransfer(); } +#if NIX_WITH_CURL_S3 +void FileTransferRequest::setupForS3() +{ + auto parsedS3 = ParsedS3URL::parse(uri.parsed()); + // Update the request URI to use HTTPS + uri = parsedS3.toHttpsUrl(); + // This gets used later in a curl setopt + awsSigV4Provider = "aws:amz:" + parsedS3.region.value_or("us-east-1") + ":s3"; + // check if the request already has pre-resolved credentials + std::optional sessionToken; + if (usernameAuth) { + debug("Using pre-resolved AWS credentials from parent process"); + sessionToken = preResolvedAwsSessionToken; + } else { + std::string profile = parsedS3.profile.value_or(""); + try { + auto creds = getAwsCredentials(profile); + usernameAuth = UsernameAuth{ + .username = creds.accessKeyId, + .password = creds.secretAccessKey, + }; + sessionToken = creds.sessionToken; + } catch (const AwsAuthError & e) { + warn("AWS authentication failed for S3 request %s: %s", uri, e.what()); + // Invalidate the cached credentials so next request will retry + invalidateAwsCredentials(profile); + // Continue without authentication - might be a public bucket + return; + } + } + if (sessionToken) + headers.emplace_back("x-amz-security-token", *sessionToken); +} +#endif + std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) { auto promise = std::make_shared>(); diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh index 16643c555..4930dc9d8 100644 --- a/src/libstore/include/nix/store/aws-creds.hh +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -57,12 +57,6 @@ AwsCredentials getAwsCredentials(const std::string & profile = ""); */ void invalidateAwsCredentials(const std::string & profile); -/** - * Clear all cached credential providers. - * Typically called during application cleanup. - */ -void clearAwsCredentialsCache(); - /** * Pre-resolve AWS credentials for S3 URLs. * Used to cache credentials in parent process before forking. diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 2f2d59036..942e05a61 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -11,6 +11,11 @@ #include "nix/util/serialise.hh" #include "nix/util/url.hh" +#include "nix/store/config.hh" +#if NIX_WITH_CURL_S3 +# include "nix/store/aws-creds.hh" +#endif + namespace nix { struct FileTransferSettings : Config @@ -77,6 +82,17 @@ extern FileTransferSettings fileTransferSettings; extern const unsigned int RETRY_TIME_MS_DEFAULT; +/** + * Username and optional password for HTTP basic authentication. + * These are used with curl's CURLOPT_USERNAME and CURLOPT_PASSWORD options + * for various protocols including HTTP, FTP, and others. + */ +struct UsernameAuth +{ + std::string username; + std::optional password; +}; + struct FileTransferRequest { ValidURL uri; @@ -92,6 +108,18 @@ struct FileTransferRequest std::optional data; std::string mimeType; std::function dataCallback; + /** + * Optional username and password for HTTP basic authentication. + * When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option. + */ + std::optional usernameAuth; +#if NIX_WITH_CURL_S3 + /** + * Pre-resolved AWS session token for S3 requests. + * When provided along with usernameAuth, this will be used instead of fetching fresh credentials. + */ + std::optional preResolvedAwsSessionToken; +#endif FileTransferRequest(ValidURL uri) : uri(std::move(uri)) @@ -103,6 +131,13 @@ struct FileTransferRequest { return data ? "upload" : "download"; } + +#if NIX_WITH_CURL_S3 +private: + friend struct curlFileTransfer; + void setupForS3(); + std::optional awsSigV4Provider; +#endif }; struct FileTransferResult