1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-08 19:46:02 +01:00

feat(libstore/s3-binary-cache-store): add multipart upload config settings

Add three configuration settings to `S3BinaryCacheStoreConfig` to control
multipart upload behavior:

- `bool multipart-upload` (default `false`): Enable/disable multipart uploads
- `uint64_t multipart-chunk-size` (default 5 MiB): Size of each upload part
- `uint64_t multipart-threshold` (default 100 MiB): Minimum file size for multipart

The feature is disabled by default.
This commit is contained in:
Bernardo Meurer Costa 2025-10-21 06:19:17 +00:00 committed by Bernardo Meurer
parent 2d83bc6b83
commit bf947bfc26
2 changed files with 55 additions and 0 deletions

View file

@ -61,6 +61,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
> addressing instead of virtual host based addressing.
)"};
const Setting<bool> multipartUpload{
this,
false,
"multipart-upload",
R"(
Whether to use multipart uploads for large files. When enabled,
files exceeding the multipart threshold will be uploaded in
multiple parts, which is required for files larger than 5 GiB and
can improve performance and reliability for large uploads.
)"};
const Setting<uint64_t> multipartChunkSize{
this,
5 * 1024 * 1024,
"multipart-chunk-size",
R"(
The size (in bytes) of each part in multipart uploads. Must be
at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the
number of requests but use more memory. Default is 5 MiB.
)",
{"buffer-size"}};
const Setting<uint64_t> multipartThreshold{
this,
100 * 1024 * 1024,
"multipart-threshold",
R"(
The minimum file size (in bytes) for using multipart uploads.
Files smaller than this threshold will use regular PUT requests.
Default is 100 MiB. Only takes effect when multipart-upload is enabled.
)"};
/**
* Set of settings that are part of the S3 URI itself.
* These are needed for region specification and other S3-specific settings.

View file

@ -15,6 +15,7 @@ namespace nix {
MakeError(UploadToS3, Error);
static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB
static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
@ -253,6 +254,28 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
cacheUri.query[key] = value;
}
}
if (multipartChunkSize < AWS_MIN_PART_SIZE) {
throw UsageError(
"multipart-chunk-size must be at least %s, got %s",
renderSize(AWS_MIN_PART_SIZE),
renderSize(multipartChunkSize.get()));
}
if (multipartChunkSize > AWS_MAX_PART_SIZE) {
throw UsageError(
"multipart-chunk-size must be at most %s, got %s",
renderSize(AWS_MAX_PART_SIZE),
renderSize(multipartChunkSize.get()));
}
if (multipartUpload && multipartThreshold < multipartChunkSize) {
warn(
"multipart-threshold (%s) is less than multipart-chunk-size (%s), "
"which may result in single-part multipart uploads",
renderSize(multipartThreshold.get()),
renderSize(multipartChunkSize.get()));
}
}
std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const