mirror of
https://github.com/NixOS/nix.git
synced 2025-11-08 19:46:02 +01:00
feat(libstore/s3-binary-cache-store): add multipart upload config settings
Add three configuration settings to `S3BinaryCacheStoreConfig` to control multipart upload behavior: - `bool multipart-upload` (default `false`): Enable/disable multipart uploads - `uint64_t multipart-chunk-size` (default 5 MiB): Size of each upload part - `uint64_t multipart-threshold` (default 100 MiB): Minimum file size for multipart The feature is disabled by default.
This commit is contained in:
parent
2d83bc6b83
commit
bf947bfc26
2 changed files with 55 additions and 0 deletions
|
|
@ -61,6 +61,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||||
> addressing instead of virtual host based addressing.
|
> addressing instead of virtual host based addressing.
|
||||||
)"};
|
)"};
|
||||||
|
|
||||||
|
const Setting<bool> multipartUpload{
|
||||||
|
this,
|
||||||
|
false,
|
||||||
|
"multipart-upload",
|
||||||
|
R"(
|
||||||
|
Whether to use multipart uploads for large files. When enabled,
|
||||||
|
files exceeding the multipart threshold will be uploaded in
|
||||||
|
multiple parts, which is required for files larger than 5 GiB and
|
||||||
|
can improve performance and reliability for large uploads.
|
||||||
|
)"};
|
||||||
|
|
||||||
|
const Setting<uint64_t> multipartChunkSize{
|
||||||
|
this,
|
||||||
|
5 * 1024 * 1024,
|
||||||
|
"multipart-chunk-size",
|
||||||
|
R"(
|
||||||
|
The size (in bytes) of each part in multipart uploads. Must be
|
||||||
|
at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the
|
||||||
|
number of requests but use more memory. Default is 5 MiB.
|
||||||
|
)",
|
||||||
|
{"buffer-size"}};
|
||||||
|
|
||||||
|
const Setting<uint64_t> multipartThreshold{
|
||||||
|
this,
|
||||||
|
100 * 1024 * 1024,
|
||||||
|
"multipart-threshold",
|
||||||
|
R"(
|
||||||
|
The minimum file size (in bytes) for using multipart uploads.
|
||||||
|
Files smaller than this threshold will use regular PUT requests.
|
||||||
|
Default is 100 MiB. Only takes effect when multipart-upload is enabled.
|
||||||
|
)"};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set of settings that are part of the S3 URI itself.
|
* Set of settings that are part of the S3 URI itself.
|
||||||
* These are needed for region specification and other S3-specific settings.
|
* These are needed for region specification and other S3-specific settings.
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ namespace nix {
|
||||||
|
|
||||||
MakeError(UploadToS3, Error);
|
MakeError(UploadToS3, Error);
|
||||||
|
|
||||||
|
static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB
|
||||||
static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB
|
static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB
|
||||||
|
|
||||||
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
|
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
|
||||||
|
|
@ -253,6 +254,28 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
|
||||||
cacheUri.query[key] = value;
|
cacheUri.query[key] = value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (multipartChunkSize < AWS_MIN_PART_SIZE) {
|
||||||
|
throw UsageError(
|
||||||
|
"multipart-chunk-size must be at least %s, got %s",
|
||||||
|
renderSize(AWS_MIN_PART_SIZE),
|
||||||
|
renderSize(multipartChunkSize.get()));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (multipartChunkSize > AWS_MAX_PART_SIZE) {
|
||||||
|
throw UsageError(
|
||||||
|
"multipart-chunk-size must be at most %s, got %s",
|
||||||
|
renderSize(AWS_MAX_PART_SIZE),
|
||||||
|
renderSize(multipartChunkSize.get()));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (multipartUpload && multipartThreshold < multipartChunkSize) {
|
||||||
|
warn(
|
||||||
|
"multipart-threshold (%s) is less than multipart-chunk-size (%s), "
|
||||||
|
"which may result in single-part multipart uploads",
|
||||||
|
renderSize(multipartThreshold.get()),
|
||||||
|
renderSize(multipartChunkSize.get()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const
|
std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue