nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-09 03:56:01 +01:00
Code Activity

Apply clang-format universally.

Browse source 
* It is tough to contribute to a project that doesn't use a formatter,
* It is extra hard to contribute to a project which has configured the formatter, but ignores it for some files
* Code formatting makes it harder to hide obscure / weird bugs by accident or on purpose,

Let's rip the bandaid off?

Note that PRs currently in flight should be able to be merged relatively easily by applying `clang-format` to their tip prior to merge.
This commit is contained in:
Graham Christensen 2025-07-18 12:47:27 -04:00
parent 41bf87ec70
commit e4f62e4608
587 changed files with 23258 additions and 23135 deletions
Download patch file Download diff file Expand all files Collapse all files

289
src/libstore/s3-binary-cache-store.cc
View file

@ -2,33 +2,33 @@
#if NIX_WITH_S3_SUPPORT
#include <assert.h>
# include <assert.h>
#include "nix/store/s3.hh"
#include "nix/store/nar-info.hh"
#include "nix/store/nar-info-disk-cache.hh"
#include "nix/store/globals.hh"
#include "nix/util/compression.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-registration.hh"
# include "nix/store/s3.hh"
# include "nix/store/nar-info.hh"
# include "nix/store/nar-info-disk-cache.hh"
# include "nix/store/globals.hh"
# include "nix/util/compression.hh"
# include "nix/store/filetransfer.hh"
# include "nix/util/signals.hh"
# include "nix/store/store-registration.hh"
#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
#include <aws/core/auth/AWSCredentialsProvider.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/client/ClientConfiguration.h>
#include <aws/core/client/DefaultRetryStrategy.h>
#include <aws/core/utils/logging/FormattedLogSystem.h>
#include <aws/core/utils/logging/LogMacros.h>
#include <aws/core/utils/threading/Executor.h>
#include <aws/identity-management/auth/STSProfileCredentialsProvider.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/transfer/TransferManager.h>
# include <aws/core/Aws.h>
# include <aws/core/VersionConfig.h>
# include <aws/core/auth/AWSCredentialsProvider.h>
# include <aws/core/auth/AWSCredentialsProviderChain.h>
# include <aws/core/client/ClientConfiguration.h>
# include <aws/core/client/DefaultRetryStrategy.h>
# include <aws/core/utils/logging/FormattedLogSystem.h>
# include <aws/core/utils/logging/LogMacros.h>
# include <aws/core/utils/threading/Executor.h>
# include <aws/identity-management/auth/STSProfileCredentialsProvider.h>
# include <aws/s3/S3Client.h>
# include <aws/s3/model/GetObjectRequest.h>
# include <aws/s3/model/HeadObjectRequest.h>
# include <aws/s3/model/ListObjectsRequest.h>
# include <aws/s3/model/PutObjectRequest.h>
# include <aws/transfer/TransferManager.h>
using namespace Aws::Transfer;
@ -40,8 +40,10 @@ struct S3Error : public Error
Aws::String exceptionName;
template<typename... Args>
S3Error(Aws::S3::S3Errors err, Aws::String exceptionName, const Args & ... args)
: Error(args...), err(err), exceptionName(exceptionName) { };
S3Error(Aws::S3::S3Errors err, Aws::String exceptionName, const Args &... args)
: Error(args...)
, err(err)
, exceptionName(exceptionName){};
};
/* Helper: given an Outcome<R, E>, return R in case of success, or
@ -53,11 +55,7 @@ R && checkAws(std::string_view s, Aws::Utils::Outcome<R, E> && outcome)
throw S3Error(
outcome.GetError().GetErrorType(),
outcome.GetError().GetExceptionName(),
fmt(
"%s: %s (request id: %s)",
s,
outcome.GetError().GetMessage(),
outcome.GetError().GetRequestId()));
fmt("%s: %s (request id: %s)", s, outcome.GetError().GetMessage(), outcome.GetError().GetRequestId()));
return outcome.GetResultWithOwnership();
}
@ -70,9 +68,9 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
debug("AWS: %s", chomp(statement));
}
#if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115)
# if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115)
void Flush() override {}
#endif
# endif
};
/* Retrieve the credentials from the list of AWS default providers, with the addition of the STS creds provider. This
@ -110,9 +108,7 @@ static void initAWS()
if (verbosity >= lvlDebug) {
options.loggingOptions.logLevel =
verbosity == lvlDebug
? Aws::Utils::Logging::LogLevel::Debug
: Aws::Utils::Logging::LogLevel::Trace;
verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug : Aws::Utils::Logging::LogLevel::Trace;
options.loggingOptions.logger_create_fn = [options]() {
return std::make_shared<AwsLogger>(options.loggingOptions.logLevel);
};
@ -123,32 +119,31 @@ static void initAWS()
}
S3Helper::S3Helper(
const std::string & profile,
const std::string & region,
const std::string & scheme,
const std::string & endpoint)
const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint)
: config(makeConfig(region, scheme, endpoint))
, client(make_ref<Aws::S3::S3Client>(
std::make_shared<CustomAwsCredentialsProviderChain>(profile),
*config,
#if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3
false,
#else
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
#endif
endpoint.empty()))
, client(
make_ref<Aws::S3::S3Client>(
std::make_shared<CustomAwsCredentialsProviderChain>(profile),
*config,
# if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3
false,
# else
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
# endif
endpoint.empty()))
{
}
/* Log AWS retries. */
class RetryStrategy : public Aws::Client::DefaultRetryStrategy
{
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors> & error, long attemptedRetries) const override
{
checkInterrupt();
auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
if (retry)
printError("AWS error '%s' (%s; request id: %s), will retry in %d ms",
printError(
"AWS error '%s' (%s; request id: %s), will retry in %d ms",
error.GetExceptionName(),
error.GetMessage(),
error.GetRequestId(),
@ -157,10 +152,8 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
}
};
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
const std::string & region,
const std::string & scheme,
const std::string & endpoint)
ref<Aws::Client::ClientConfiguration>
S3Helper::makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint)
{
initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>();
@ -179,38 +172,30 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
return res;
}
S3Helper::FileTransferResult S3Helper::getObject(
const std::string & bucketName, const std::string & key)
S3Helper::FileTransferResult S3Helper::getObject(const std::string & bucketName, const std::string & key)
{
std::string uri = "s3://" + bucketName + "/" + key;
Activity act(*logger, lvlTalkative, actFileTransfer,
fmt("downloading '%s'", uri),
Logger::Fields{uri}, getCurActivity());
Activity act(
*logger, lvlTalkative, actFileTransfer, fmt("downloading '%s'", uri), Logger::Fields{uri}, getCurActivity());
auto request =
Aws::S3::Model::GetObjectRequest()
.WithBucket(bucketName)
.WithKey(key);
auto request = Aws::S3::Model::GetObjectRequest().WithBucket(bucketName).WithKey(key);
request.SetResponseStreamFactory([&]() {
return Aws::New<std::stringstream>("STRINGSTREAM");
});
request.SetResponseStreamFactory([&]() { return Aws::New<std::stringstream>("STRINGSTREAM"); });
size_t bytesDone = 0;
size_t bytesExpected = 0;
request.SetDataReceivedEventHandler([&](const Aws::Http::HttpRequest * req, Aws::Http::HttpResponse * resp, long long l) {
if (!bytesExpected && resp->HasHeader("Content-Length")) {
if (auto length = string2Int<size_t>(resp->GetHeader("Content-Length"))) {
bytesExpected = *length;
request.SetDataReceivedEventHandler(
[&](const Aws::Http::HttpRequest * req, Aws::Http::HttpResponse * resp, long long l) {
if (!bytesExpected && resp->HasHeader("Content-Length")) {
if (auto length = string2Int<size_t>(resp->GetHeader("Content-Length"))) {
bytesExpected = *length;
}
}
}
bytesDone += l;
act.progress(bytesDone, bytesExpected);
});
bytesDone += l;
act.progress(bytesDone, bytesExpected);
});
request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) {
return !isInterrupted();
});
request.SetContinueRequestHandler([](const Aws::Http::HttpRequest *) { return !isInterrupted(); });
FileTransferResult res;
@ -218,23 +203,20 @@ S3Helper::FileTransferResult S3Helper::getObject(
try {
auto result = checkAws(fmt("AWS error fetching '%s'", key),
client->GetObject(request));
auto result = checkAws(fmt("AWS error fetching '%s'", key), client->GetObject(request));
act.progress(result.GetContentLength(), result.GetContentLength());
res.data = decompress(result.GetContentEncoding(),
dynamic_cast<std::stringstream &>(result.GetBody()).str());
res.data = decompress(result.GetContentEncoding(), dynamic_cast<std::stringstream &>(result.GetBody()).str());
} catch (S3Error & e) {
if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) &&
(e.err != Aws::S3::S3Errors::ACCESS_DENIED) &&
if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) && (e.err != Aws::S3::S3Errors::ACCESS_DENIED) &&
// Expired tokens are not really an error, more of a caching problem. Should be treated same as 403.
//
// AWS unwilling to provide a specific error type for the situation (https://github.com/aws/aws-sdk-cpp/issues/1843)
// so use this hack
(e.exceptionName != "ExpiredToken")
) throw;
// AWS unwilling to provide a specific error type for the situation
// (https://github.com/aws/aws-sdk-cpp/issues/1843) so use this hack
(e.exceptionName != "ExpiredToken"))
throw;
}
auto now2 = std::chrono::steady_clock::now();
@ -244,11 +226,8 @@ S3Helper::FileTransferResult S3Helper::getObject(
return res;
}
S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
std::string_view uriScheme,
std::string_view bucketName,
const Params & params)
std::string_view uriScheme, std::string_view bucketName, const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, bucketName(bucketName)
@ -262,20 +241,19 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme);
}
S3BinaryCacheStore::S3BinaryCacheStore(ref<Config> config)
: BinaryCacheStore(*config)
, config{config}
{ }
{
}
std::string S3BinaryCacheStoreConfig::doc()
{
return
#include "s3-binary-cache-store.md"
;
# include "s3-binary-cache-store.md"
;
}
struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
{
Stats stats;
@ -303,8 +281,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
config->priority.setDefault(cacheInfo->priority);
} else {
BinaryCacheStore::init();
diskCache->createCache(
getUri(), config->storeDir, config->wantMassQuery, config->priority);
diskCache->createCache(getUri(), config->storeDir, config->wantMassQuery, config->priority);
}
}
@ -332,17 +309,15 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
stats.head++;
auto res = s3Helper.client->HeadObject(
Aws::S3::Model::HeadObjectRequest()
.WithBucket(config->bucketName)
.WithKey(path));
Aws::S3::Model::HeadObjectRequest().WithBucket(config->bucketName).WithKey(path));
if (!res.IsSuccess()) {
auto & error = res.GetError();
if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
|| error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
// Expired tokens are not really an error, more of a caching problem. Should be treated same as 403.
// AWS unwilling to provide a specific error type for the situation (https://github.com/aws/aws-sdk-cpp/issues/1843)
// so use this hack
// AWS unwilling to provide a specific error type for the situation
// (https://github.com/aws/aws-sdk-cpp/issues/1843) so use this hack
|| (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN && error.GetExceptionName() == "ExpiredToken")
// If bucket listing is disabled, 404s turn into 403s
|| error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
@ -373,29 +348,31 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
cv.wait(lk);
}
AsyncContext(const Activity & act) : act(act) {}
AsyncContext(const Activity & act)
: act(act)
{
}
};
void uploadFile(const std::string & path,
void uploadFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
const std::string & contentEncoding)
{
std::string uri = "s3://" + config->bucketName + "/" + path;
Activity act(*logger, lvlTalkative, actFileTransfer,
fmt("uploading '%s'", uri),
Logger::Fields{uri}, getCurActivity());
Activity act(
*logger, lvlTalkative, actFileTransfer, fmt("uploading '%s'", uri), Logger::Fields{uri}, getCurActivity());
istream->seekg(0, istream->end);
auto size = istream->tellg();
istream->seekg(0, istream->beg);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> executor =
std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
std::call_once(transferManagerCreated, [&]()
{
std::call_once(transferManagerCreated, [&]() {
if (config->multipartUpload) {
TransferManagerConfiguration transferConfig(executor.get());
@ -404,8 +381,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
transferConfig.uploadProgressCallback =
[](const TransferManager * transferManager,
const std::shared_ptr<const TransferHandle> & transferHandle)
{
const std::shared_ptr<const TransferHandle> & transferHandle) {
auto context = std::dynamic_pointer_cast<const AsyncContext>(transferHandle->GetContext());
size_t bytesDone = transferHandle->GetBytesTransferred();
size_t bytesTotal = transferHandle->GetBytesTotalSize();
@ -418,8 +394,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
};
transferConfig.transferStatusUpdatedCallback =
[](const TransferManager * transferManager,
const std::shared_ptr<const TransferHandle> & transferHandle)
{
const std::shared_ptr<const TransferHandle> & transferHandle) {
auto context = std::dynamic_pointer_cast<const AsyncContext>(transferHandle->GetContext());
context->notify();
};
@ -438,11 +413,13 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
throw Error("setting a content encoding is not supported with S3 multi-part uploads");
auto context = std::make_shared<AsyncContext>(act);
std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile(
istream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
context /*, contentEncoding */);
std::shared_ptr<TransferHandle> transferHandle = transferManager->UploadFile(
istream,
bucketName,
path,
mimeType,
Aws::Map<Aws::String, Aws::String>(),
context /*, contentEncoding */);
TransferStatus status = transferHandle->GetStatus();
while (status == TransferStatus::IN_PROGRESS || status == TransferStatus::NOT_STARTED) {
@ -457,20 +434,19 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
act.progress(transferHandle->GetBytesTransferred(), transferHandle->GetBytesTotalSize());
if (status == TransferStatus::FAILED)
throw Error("AWS error: failed to upload 's3://%s/%s': %s",
bucketName, path, transferHandle->GetLastError().GetMessage());
throw Error(
"AWS error: failed to upload 's3://%s/%s': %s",
bucketName,
path,
transferHandle->GetLastError().GetMessage());
if (status != TransferStatus::COMPLETED)
throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state",
bucketName, path);
throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", bucketName, path);
} else {
act.progress(0, size);
auto request =
Aws::S3::Model::PutObjectRequest()
.WithBucket(bucketName)
.WithKey(path);
auto request = Aws::S3::Model::PutObjectRequest().WithBucket(bucketName).WithKey(path);
size_t bytesSent = 0;
request.SetDataSentEventHandler([&](const Aws::Http::HttpRequest * req, long long l) {
@ -478,9 +454,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
act.progress(bytesSent, size);
});
request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) {
return !isInterrupted();
});
request.SetContinueRequestHandler([](const Aws::Http::HttpRequest *) { return !isInterrupted(); });
request.SetContentType(mimeType);
@ -489,32 +463,28 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
request.SetBody(istream);
auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request));
auto result = checkAws(fmt("AWS error uploading '%s'", path), s3Helper.client->PutObject(request));
act.progress(size, size);
}
auto now2 = std::chrono::steady_clock::now();
auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, path, size, duration);
stats.putTimeMs += duration;
stats.putBytes += std::max(size, (decltype(size)) 0);
stats.put++;
}
void upsertFile(const std::string & path,
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto compress = [&](std::string compression)
{
auto compress = [&](std::string compression) {
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(compressed));
};
@ -540,8 +510,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
stats.getTimeMs += res.durationMs;
if (res.data) {
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
config->bucketName, path, res.data->size(), res.durationMs);
printTalkative(
"downloaded 's3://%s/%s' (%d bytes) in %d ms",
config->bucketName,
path,
res.data->size(),
res.durationMs);
sink(*res.data);
} else
@ -558,21 +532,19 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
do {
debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker);
auto res = checkAws(fmt("AWS error listing bucket '%s'", bucketName),
auto res = checkAws(
fmt("AWS error listing bucket '%s'", bucketName),
s3Helper.client->ListObjects(
Aws::S3::Model::ListObjectsRequest()
.WithBucket(bucketName)
.WithDelimiter("/")
.WithMarker(marker)));
Aws::S3::Model::ListObjectsRequest().WithBucket(bucketName).WithDelimiter("/").WithMarker(marker)));
auto & contents = res.GetContents();
debug("got %d keys, next marker '%s'",
contents.size(), res.GetNextMarker());
debug("got %d keys, next marker '%s'", contents.size(), res.GetNextMarker());
for (const auto & object : contents) {
auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
if (key.size() != 40 || !hasSuffix(key, ".narinfo"))
continue;
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
}
@ -595,16 +567,15 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
ref<Store> S3BinaryCacheStoreImpl::Config::openStore() const
{
auto store = make_ref<S3BinaryCacheStoreImpl>(ref{
// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<S3BinaryCacheStore::Config>(shared_from_this())
});
auto store =
make_ref<S3BinaryCacheStoreImpl>(ref{// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<S3BinaryCacheStore::Config>(shared_from_this())});
store->init();
return store;
}
static RegisterStoreImplementation<S3BinaryCacheStoreImpl::Config> regS3BinaryCacheStore;
}
} // namespace nix
#endif