mirror of
https://github.com/NixOS/nix.git
synced 2025-11-08 19:46:02 +01:00
feat(libstore/s3-binary-cache-store): add multipart upload config settings
Add three configuration settings to `S3BinaryCacheStoreConfig` to control multipart upload behavior: - `bool multipart-upload` (default `false`): Enable/disable multipart uploads - `uint64_t multipart-chunk-size` (default 5 MiB): Size of each upload part - `uint64_t multipart-threshold` (default 100 MiB): Minimum file size for multipart The feature is disabled by default.
This commit is contained in:
parent
2d83bc6b83
commit
bf947bfc26
2 changed files with 55 additions and 0 deletions
|
|
@ -61,6 +61,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
|||
> addressing instead of virtual host based addressing.
|
||||
)"};
|
||||
|
||||
const Setting<bool> multipartUpload{
|
||||
this,
|
||||
false,
|
||||
"multipart-upload",
|
||||
R"(
|
||||
Whether to use multipart uploads for large files. When enabled,
|
||||
files exceeding the multipart threshold will be uploaded in
|
||||
multiple parts, which is required for files larger than 5 GiB and
|
||||
can improve performance and reliability for large uploads.
|
||||
)"};
|
||||
|
||||
const Setting<uint64_t> multipartChunkSize{
|
||||
this,
|
||||
5 * 1024 * 1024,
|
||||
"multipart-chunk-size",
|
||||
R"(
|
||||
The size (in bytes) of each part in multipart uploads. Must be
|
||||
at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the
|
||||
number of requests but use more memory. Default is 5 MiB.
|
||||
)",
|
||||
{"buffer-size"}};
|
||||
|
||||
const Setting<uint64_t> multipartThreshold{
|
||||
this,
|
||||
100 * 1024 * 1024,
|
||||
"multipart-threshold",
|
||||
R"(
|
||||
The minimum file size (in bytes) for using multipart uploads.
|
||||
Files smaller than this threshold will use regular PUT requests.
|
||||
Default is 100 MiB. Only takes effect when multipart-upload is enabled.
|
||||
)"};
|
||||
|
||||
/**
|
||||
* Set of settings that are part of the S3 URI itself.
|
||||
* These are needed for region specification and other S3-specific settings.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ namespace nix {
|
|||
|
||||
MakeError(UploadToS3, Error);
|
||||
|
||||
static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB
|
||||
static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB
|
||||
|
||||
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
|
||||
|
|
@ -253,6 +254,28 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
|
|||
cacheUri.query[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
if (multipartChunkSize < AWS_MIN_PART_SIZE) {
|
||||
throw UsageError(
|
||||
"multipart-chunk-size must be at least %s, got %s",
|
||||
renderSize(AWS_MIN_PART_SIZE),
|
||||
renderSize(multipartChunkSize.get()));
|
||||
}
|
||||
|
||||
if (multipartChunkSize > AWS_MAX_PART_SIZE) {
|
||||
throw UsageError(
|
||||
"multipart-chunk-size must be at most %s, got %s",
|
||||
renderSize(AWS_MAX_PART_SIZE),
|
||||
renderSize(multipartChunkSize.get()));
|
||||
}
|
||||
|
||||
if (multipartUpload && multipartThreshold < multipartChunkSize) {
|
||||
warn(
|
||||
"multipart-threshold (%s) is less than multipart-chunk-size (%s), "
|
||||
"which may result in single-part multipart uploads",
|
||||
renderSize(multipartThreshold.get()),
|
||||
renderSize(multipartChunkSize.get()));
|
||||
}
|
||||
}
|
||||
|
||||
std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue