libstore: Put all the AWS credentials logic behind interface class AwsCredentialProvider

This makes it so we don't need to rely on global variables and hacky destructors to
clean up another global variable. Just putting it in the correct order in the class
is more than enough.

src/libstore/aws-creds.cc

@@ -24,50 +24,6 @@ namespace nix {
 
 namespace {
 
-// Global credential provider cache using boost's concurrent map
-// Key: profile name (empty string for default profile)
-using CredentialProviderCache =
-    boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>;
-
-static CredentialProviderCache credentialProviderCache;
-
-/**
- * Clear all cached credential providers.
- * Called automatically by CrtWrapper destructor during static destruction.
- */
-static void clearAwsCredentialsCache()
-{
-    credentialProviderCache.clear();
-}
-
-static void initAwsCrt()
-{
-    struct CrtWrapper
-    {
-        Aws::Crt::ApiHandle apiHandle;
-
-        CrtWrapper()
-        {
-            apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast<FILE *>(nullptr));
-        }
-
-        ~CrtWrapper()
-        {
-            try {
-                // CRITICAL: Clear credential provider cache BEFORE AWS CRT shuts down
-                // This ensures all providers (which hold references to ClientBootstrap)
-                // are destroyed while AWS CRT is still valid
-                clearAwsCredentialsCache();
-                // Now it's safe for ApiHandle destructor to run
-            } catch (...) {
-                ignoreExceptionInDestructor();
-            }
-        }
-    };
-
-    static CrtWrapper crt;
-}
-
 static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
 {
     if (!provider || !provider->IsValid()) {
@@ -113,7 +69,35 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth:
 
 } // anonymous namespace
 
-AwsCredentials getAwsCredentials(const std::string & profile)
+class AwsCredentialProviderImpl : public AwsCredentialProvider
+{
+public:
+    AwsCredentialProviderImpl()
+    {
+        apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast<FILE *>(nullptr));
+    }
+
+    AwsCredentials getCredentialsRaw(const std::string & profile);
+
+    AwsCredentials getCredentials(const ParsedS3URL & url) override
+    {
+        auto profile = url.profile.value_or("");
+        try {
+            return getCredentialsRaw(profile);
+        } catch (AwsAuthError & e) {
+            warn("AWS authentication failed for S3 request %s: %s", url.toHttpsUrl(), e.what());
+            credentialProviderCache.erase(profile);
+            throw;
+        }
+    }
+
+private:
+    Aws::Crt::ApiHandle apiHandle;
+    boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
+        credentialProviderCache;
+};
+
+AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
 {
     // Get or create credential provider with caching
     std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider;
@@ -132,8 +116,6 @@ AwsCredentials getAwsCredentials(const std::string & profile)
                 profile.empty() ? "(default)" : profile.c_str());
 
             try {
-                initAwsCrt();
-
                 if (profile.empty()) {
                     Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
                     config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
@@ -173,17 +155,15 @@ AwsCredentials getAwsCredentials(const std::string & profile)
     return getCredentialsFromProvider(provider);
 }
 
-void invalidateAwsCredentials(const std::string & profile)
+ref<AwsCredentialProvider> makeAwsCredentialsProvider()
 {
-    credentialProviderCache.erase(profile);
+    return make_ref<AwsCredentialProviderImpl>();
 }
 
-AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url)
+ref<AwsCredentialProvider> getAwsCredentialsProvider()
 {
-    std::string profile = s3Url.profile.value_or("");
+    static auto instance = makeAwsCredentialsProvider();
-
+    return instance;
-    // Get credentials (automatically cached)
-    return getAwsCredentials(profile);
 }
 
 } // namespace nix

src/libstore/filetransfer.cc

@@ -883,22 +883,12 @@ void FileTransferRequest::setupForS3()
     if (usernameAuth) {
         debug("Using pre-resolved AWS credentials from parent process");
         sessionToken = preResolvedAwsSessionToken;
-    } else {
+    } else if (auto creds = getAwsCredentialsProvider()->maybeGetCredentials(parsedS3)) {
-        std::string profile = parsedS3.profile.value_or("");
-        try {
-            auto creds = getAwsCredentials(profile);
         usernameAuth = UsernameAuth{
-                .username = creds.accessKeyId,
+            .username = creds->accessKeyId,
-                .password = creds.secretAccessKey,
+            .password = creds->secretAccessKey,
         };
-            sessionToken = creds.sessionToken;
+        sessionToken = creds->sessionToken;
-        } catch (const AwsAuthError & e) {
-            warn("AWS authentication failed for S3 request %s: %s", uri, e.what());
-            // Invalidate the cached credentials so next request will retry
-            invalidateAwsCredentials(profile);
-            // Continue without authentication - might be a public bucket
-            return;
-        }
     }
     if (sessionToken)
         headers.emplace_back("x-amz-security-token", *sessionToken);

src/libstore/include/nix/store/aws-creds.hh

@@ -5,6 +5,7 @@
 #if NIX_WITH_AWS_AUTH
 
 #  include "nix/store/s3-url.hh"
+#  include "nix/util/ref.hh"
 #  include "nix/util/error.hh"
 
 #  include <memory>
@@ -38,30 +39,39 @@ struct AwsCredentials
  */
 MakeError(AwsAuthError, Error);
 
-/**
+class AwsCredentialProvider
- * Get AWS credentials for the given profile.
+{
- * This function automatically caches credential providers to avoid
+public:
- * creating multiple providers for the same profile.
+    /**
+     * Get AWS credentials for the given URL.
      *
- * @param profile The AWS profile name (empty string for default profile)
+     * @param url The S3 url to get the credentials for
      * @return AWS credentials
      * @throws AwsAuthError if credentials cannot be resolved
      */
-AwsCredentials getAwsCredentials(const std::string & profile = "");
+    virtual AwsCredentials getCredentials(const ParsedS3URL & url) = 0;
+
+    std::optional<AwsCredentials> maybeGetCredentials(const ParsedS3URL & url)
+    {
+        try {
+            return getCredentials(url);
+        } catch (AwsAuthError & e) {
+            return std::nullopt;
+        }
+    }
+
+    virtual ~AwsCredentialProvider() {}
+};
 
 /**
- * Invalidate cached credentials for a profile (e.g., on authentication failure).
+ * Create a new instancee of AwsCredentialProvider.
- * The next request for this profile will create a new provider.
- *
- * @param profile The AWS profile name to invalidate
  */
-void invalidateAwsCredentials(const std::string & profile);
+ref<AwsCredentialProvider> makeAwsCredentialsProvider();
 
 /**
- * Pre-resolve AWS credentials for S3 URLs.
+ * Get a reference to the global AwsCredentialProvider.
- * Used to cache credentials in parent process before forking.
  */
-AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url);
+ref<AwsCredentialProvider> getAwsCredentialsProvider();
 
 } // namespace nix
 #endif

src/libstore/unix/build/derivation-builder.cc

@@ -958,7 +958,7 @@ std::optional<AwsCredentials> DerivationBuilderImpl::preResolveAwsCredentials()
                     auto s3Url = ParsedS3URL::parse(parsedUrl);
 
                     // Use the preResolveAwsCredentials from aws-creds
-                    auto credentials = nix::preResolveAwsCredentials(s3Url);
+                    auto credentials = getAwsCredentialsProvider()->getCredentials(s3Url);
                     debug("Successfully pre-resolved AWS credentials in parent process");
                     return credentials;
                 }