mirror of
https://github.com/NixOS/nix.git
synced 2025-11-09 12:06:01 +01:00
feat(libstore): add AWS CRT-based credential infrastructure
Add lightweight AWS credential resolution using AWS CRT (Common Runtime) instead of the full AWS SDK. This provides credential management for the upcoming curl-based S3 implementation.
This commit is contained in:
parent
30a6cbe90b
commit
a4e792cba7
4 changed files with 253 additions and 0 deletions
178
src/libstore/aws-creds.cc
Normal file
178
src/libstore/aws-creds.cc
Normal file
|
|
@ -0,0 +1,178 @@
|
||||||
|
#include "nix/store/aws-creds.hh"
|
||||||
|
|
||||||
|
#if NIX_WITH_S3_SUPPORT
|
||||||
|
|
||||||
|
# include <aws/crt/Types.h>
|
||||||
|
# include "nix/store/s3-url.hh"
|
||||||
|
# include "nix/util/finally.hh"
|
||||||
|
# include "nix/util/logging.hh"
|
||||||
|
# include "nix/util/url.hh"
|
||||||
|
# include "nix/util/util.hh"
|
||||||
|
|
||||||
|
# include <aws/crt/Api.h>
|
||||||
|
# include <aws/crt/auth/Credentials.h>
|
||||||
|
# include <aws/crt/io/Bootstrap.h>
|
||||||
|
|
||||||
|
# include <boost/unordered/concurrent_flat_map.hpp>
|
||||||
|
|
||||||
|
# include <chrono>
|
||||||
|
# include <future>
|
||||||
|
# include <memory>
|
||||||
|
# include <unistd.h>
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
static void initAwsCrt()
|
||||||
|
{
|
||||||
|
struct CrtWrapper
|
||||||
|
{
|
||||||
|
Aws::Crt::ApiHandle apiHandle;
|
||||||
|
|
||||||
|
CrtWrapper()
|
||||||
|
{
|
||||||
|
apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast<FILE *>(nullptr));
|
||||||
|
}
|
||||||
|
|
||||||
|
~CrtWrapper()
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
// CRITICAL: Clear credential provider cache BEFORE AWS CRT shuts down
|
||||||
|
// This ensures all providers (which hold references to ClientBootstrap)
|
||||||
|
// are destroyed while AWS CRT is still valid
|
||||||
|
clearAwsCredentialsCache();
|
||||||
|
// Now it's safe for ApiHandle destructor to run
|
||||||
|
} catch (...) {
|
||||||
|
ignoreExceptionInDestructor();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
static CrtWrapper crt;
|
||||||
|
}
|
||||||
|
|
||||||
|
static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
|
||||||
|
{
|
||||||
|
if (!provider || !provider->IsValid()) {
|
||||||
|
throw AwsAuthError("AWS credential provider is invalid");
|
||||||
|
}
|
||||||
|
|
||||||
|
auto prom = std::make_shared<std::promise<AwsCredentials>>();
|
||||||
|
auto fut = prom->get_future();
|
||||||
|
|
||||||
|
provider->GetCredentials([prom](std::shared_ptr<Aws::Crt::Auth::Credentials> credentials, int errorCode) {
|
||||||
|
if (errorCode != 0 || !credentials) {
|
||||||
|
prom->set_exception(
|
||||||
|
std::make_exception_ptr(AwsAuthError("Failed to resolve AWS credentials: error code %d", errorCode)));
|
||||||
|
} else {
|
||||||
|
auto accessKeyId = Aws::Crt::ByteCursorToStringView(credentials->GetAccessKeyId());
|
||||||
|
auto secretAccessKey = Aws::Crt::ByteCursorToStringView(credentials->GetSecretAccessKey());
|
||||||
|
auto sessionToken = Aws::Crt::ByteCursorToStringView(credentials->GetSessionToken());
|
||||||
|
|
||||||
|
std::optional<std::string> sessionTokenStr;
|
||||||
|
if (!sessionToken.empty()) {
|
||||||
|
sessionTokenStr = std::string(sessionToken.data(), sessionToken.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
prom->set_value(AwsCredentials(
|
||||||
|
std::string(accessKeyId.data(), accessKeyId.size()),
|
||||||
|
std::string(secretAccessKey.data(), secretAccessKey.size()),
|
||||||
|
sessionTokenStr));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// AWS CRT GetCredentials is asynchronous and only guarantees the callback will be
|
||||||
|
// invoked if the initial call returns success. There's no documented timeout mechanism,
|
||||||
|
// so we add a timeout to prevent indefinite hanging if the callback is never called.
|
||||||
|
auto timeout = std::chrono::seconds(30);
|
||||||
|
if (fut.wait_for(timeout) == std::future_status::timeout) {
|
||||||
|
throw AwsAuthError(
|
||||||
|
"Timeout waiting for AWS credentials (%d seconds)",
|
||||||
|
std::chrono::duration_cast<std::chrono::seconds>(timeout).count());
|
||||||
|
}
|
||||||
|
|
||||||
|
return fut.get(); // This will throw if set_exception was called
|
||||||
|
}
|
||||||
|
|
||||||
|
// Global credential provider cache using boost's concurrent map
|
||||||
|
// Key: profile name (empty string for default profile)
|
||||||
|
using CredentialProviderCache =
|
||||||
|
boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>;
|
||||||
|
|
||||||
|
static CredentialProviderCache credentialProviderCache;
|
||||||
|
|
||||||
|
} // anonymous namespace
|
||||||
|
|
||||||
|
AwsCredentials getAwsCredentials(const std::string & profile)
|
||||||
|
{
|
||||||
|
// Get or create credential provider with caching
|
||||||
|
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider;
|
||||||
|
|
||||||
|
// Try to find existing provider
|
||||||
|
credentialProviderCache.visit(profile, [&](const auto & pair) { provider = pair.second; });
|
||||||
|
|
||||||
|
if (!provider) {
|
||||||
|
// Create new provider if not found
|
||||||
|
debug(
|
||||||
|
"[pid=%d] creating new AWS credential provider for profile '%s'",
|
||||||
|
getpid(),
|
||||||
|
profile.empty() ? "(default)" : profile.c_str());
|
||||||
|
|
||||||
|
try {
|
||||||
|
initAwsCrt();
|
||||||
|
|
||||||
|
if (profile.empty()) {
|
||||||
|
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
|
||||||
|
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||||
|
provider = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
|
||||||
|
} else {
|
||||||
|
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
|
||||||
|
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||||
|
// This is safe because the underlying C library will copy this string
|
||||||
|
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
|
||||||
|
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
|
||||||
|
provider = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
|
||||||
|
}
|
||||||
|
} catch (Error & e) {
|
||||||
|
e.addTrace(
|
||||||
|
{},
|
||||||
|
"while creating AWS credentials provider for %s",
|
||||||
|
profile.empty() ? "default profile" : fmt("profile '%s'", profile));
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!provider) {
|
||||||
|
throw AwsAuthError(
|
||||||
|
"Failed to create AWS credentials provider for %s",
|
||||||
|
profile.empty() ? "default profile" : fmt("profile '%s'", profile));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert into cache (try_emplace is thread-safe and won't overwrite if another thread added it)
|
||||||
|
credentialProviderCache.try_emplace(profile, provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
return getCredentialsFromProvider(provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
void invalidateAwsCredentials(const std::string & profile)
|
||||||
|
{
|
||||||
|
credentialProviderCache.erase(profile);
|
||||||
|
}
|
||||||
|
|
||||||
|
void clearAwsCredentialsCache()
|
||||||
|
{
|
||||||
|
credentialProviderCache.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url)
|
||||||
|
{
|
||||||
|
std::string profile = s3Url.profile.value_or("");
|
||||||
|
|
||||||
|
// Get credentials (automatically cached)
|
||||||
|
return getAwsCredentials(profile);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
|
|
||||||
|
#endif
|
||||||
73
src/libstore/include/nix/store/aws-creds.hh
Normal file
73
src/libstore/include/nix/store/aws-creds.hh
Normal file
|
|
@ -0,0 +1,73 @@
|
||||||
|
#pragma once
|
||||||
|
///@file
|
||||||
|
#include "nix/store/config.hh"
|
||||||
|
|
||||||
|
#if NIX_WITH_S3_SUPPORT
|
||||||
|
|
||||||
|
# include "nix/store/s3-url.hh"
|
||||||
|
# include "nix/util/error.hh"
|
||||||
|
|
||||||
|
# include <memory>
|
||||||
|
# include <optional>
|
||||||
|
# include <string>
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AWS credentials obtained from credential providers
|
||||||
|
*/
|
||||||
|
struct AwsCredentials
|
||||||
|
{
|
||||||
|
std::string accessKeyId;
|
||||||
|
std::string secretAccessKey;
|
||||||
|
std::optional<std::string> sessionToken;
|
||||||
|
|
||||||
|
AwsCredentials(
|
||||||
|
const std::string & accessKeyId,
|
||||||
|
const std::string & secretAccessKey,
|
||||||
|
const std::optional<std::string> & sessionToken = std::nullopt)
|
||||||
|
: accessKeyId(accessKeyId)
|
||||||
|
, secretAccessKey(secretAccessKey)
|
||||||
|
, sessionToken(sessionToken)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exception thrown when AWS authentication fails
|
||||||
|
*/
|
||||||
|
MakeError(AwsAuthError, Error);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get AWS credentials for the given profile.
|
||||||
|
* This function automatically caches credential providers to avoid
|
||||||
|
* creating multiple providers for the same profile.
|
||||||
|
*
|
||||||
|
* @param profile The AWS profile name (empty string for default profile)
|
||||||
|
* @return AWS credentials
|
||||||
|
* @throws AwsAuthError if credentials cannot be resolved
|
||||||
|
*/
|
||||||
|
AwsCredentials getAwsCredentials(const std::string & profile = "");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Invalidate cached credentials for a profile (e.g., on authentication failure).
|
||||||
|
* The next request for this profile will create a new provider.
|
||||||
|
*
|
||||||
|
* @param profile The AWS profile name to invalidate
|
||||||
|
*/
|
||||||
|
void invalidateAwsCredentials(const std::string & profile);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clear all cached credential providers.
|
||||||
|
* Typically called during application cleanup.
|
||||||
|
*/
|
||||||
|
void clearAwsCredentialsCache();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pre-resolve AWS credentials for S3 URLs.
|
||||||
|
* Used to cache credentials in parent process before forking.
|
||||||
|
*/
|
||||||
|
AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url);
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
|
#endif
|
||||||
|
|
@ -10,6 +10,7 @@ config_pub_h = configure_file(
|
||||||
)
|
)
|
||||||
|
|
||||||
headers = [ config_pub_h ] + files(
|
headers = [ config_pub_h ] + files(
|
||||||
|
'aws-creds.hh',
|
||||||
'binary-cache-store.hh',
|
'binary-cache-store.hh',
|
||||||
'build-result.hh',
|
'build-result.hh',
|
||||||
'build/derivation-builder.hh',
|
'build/derivation-builder.hh',
|
||||||
|
|
|
||||||
|
|
@ -268,6 +268,7 @@ subdir('nix-meson-build-support/common')
|
||||||
subdir('nix-meson-build-support/asan-options')
|
subdir('nix-meson-build-support/asan-options')
|
||||||
|
|
||||||
sources = files(
|
sources = files(
|
||||||
|
'aws-creds.cc',
|
||||||
'binary-cache-store.cc',
|
'binary-cache-store.cc',
|
||||||
'build-result.cc',
|
'build-result.cc',
|
||||||
'build/derivation-building-goal.cc',
|
'build/derivation-building-goal.cc',
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue