From c663f7ec79aa21037dabc0df130eb6f3e98d10c4 Mon Sep 17 00:00:00 2001
From: Sergei Zimmerman <sergei@zimmerman.foo>
Date: Sun, 19 Oct 2025 21:03:13 +0300
Subject: [PATCH] libstore: Fix reentrancy in
 AwsCredentialProviderImpl::getCredentialsRaw

Old code would do very much incorrect reentrancy crimes (trying to do an
erase inside the emplace callback). This would fail miserably with an assertion
in Boost:

terminating due to unexpected unrecoverable internal error: Assertion '(!find(px))&&("reentrancy not allowed")' failed in boost::unordered::detail::foa::entry_trace::entry_trace(const void *) at include/boost/unordered/detail/foa/reentrancy_check.hpp:33

This is trivially reproduced by using any S3 URL with a non-empty profile:

nix-prefetch-url "s3://happy/crash?profile=default"
---
 src/libstore/aws-creds.cc | 82 +++++++++++++++++----------------------
 1 file changed, 36 insertions(+), 46 deletions(-)

diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc
index d58293560..ff7b0f0ef 100644
--- a/src/libstore/aws-creds.cc
+++ b/src/libstore/aws-creds.cc
@@ -96,67 +96,57 @@ public:
         }
     }
 
+    std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createProviderForProfile(const std::string & profile);
+
 private:
     Aws::Crt::ApiHandle apiHandle;
     boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
         credentialProviderCache;
 };
 
+std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
+AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
+{
+    debug(
+        "[pid=%d] creating new AWS credential provider for profile '%s'",
+        getpid(),
+        profile.empty() ? "(default)" : profile.c_str());
+
+    if (profile.empty()) {
+        Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
+        config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
+        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
+    }
+
+    Aws::Crt::Auth::CredentialsProviderProfileConfig config;
+    config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
+    // This is safe because the underlying C library will copy this string
+    // c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
+    config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
+    return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
+}
+
 AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
 {
-    // Get or create credential provider with caching
     std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider;
 
-    // Use try_emplace_and_cvisit for atomic get-or-create
-    // This prevents race conditions where multiple threads create providers
     credentialProviderCache.try_emplace_and_cvisit(
         profile,
-        nullptr, // Placeholder - will be replaced in f1 before any thread can see it
-        [&](auto & kv) {
-            // f1: Called atomically during insertion with non-const reference
-            // Other threads are blocked until we finish, so nullptr is never visible
-            debug(
-                "[pid=%d] creating new AWS credential provider for profile '%s'",
-                getpid(),
-                profile.empty() ? "(default)" : profile.c_str());
+        nullptr,
+        [&](auto & kv) { provider = kv.second = createProviderForProfile(profile); },
+        [&](const auto & kv) { provider = kv.second; });
 
-            try {
-                if (profile.empty()) {
-                    Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
-                    config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
-                    kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
-                } else {
-                    Aws::Crt::Auth::CredentialsProviderProfileConfig config;
-                    config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
-                    // This is safe because the underlying C library will copy this string
-                    // c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
-                    config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
-                    kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
-                }
-
-                if (!kv.second) {
-                    throw AwsAuthError(
-                        "Failed to create AWS credentials provider for %s",
-                        profile.empty() ? "default profile" : fmt("profile '%s'", profile));
-                }
-
-                provider = kv.second;
-            } catch (Error & e) {
-                // Exception during creation - remove the entry to allow retry
-                credentialProviderCache.erase(profile);
-                e.addTrace({}, "for AWS profile: %s", profile.empty() ? "(default)" : profile);
-                throw;
-            } catch (...) {
-                // Non-Error exception - still need to clean up
-                credentialProviderCache.erase(profile);
-                throw;
-            }
-        },
-        [&](const auto & kv) {
-            // f2: Called if key already exists (const reference)
-            provider = kv.second;
+    if (!provider) {
+        credentialProviderCache.erase_if(profile, [](const auto & kv) {
+            [[maybe_unused]] auto [_, provider] = kv;
+            return !provider;
         });
 
+        throw AwsAuthError(
+            "Failed to create AWS credentials provider for %s",
+            profile.empty() ? "default profile" : fmt("profile '%s'", profile));
+    }
+
     return getCredentialsFromProvider(provider);
 }