1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-11 13:06:01 +01:00

feat(libstore): add S3 storage class support

Add support for configuring S3 storage class via the storage-class
parameter for S3BinaryCacheStore. This allows users to optimize costs
by selecting appropriate storage tiers (STANDARD, GLACIER,
INTELLIGENT_TIERING, etc.) based on access patterns.

The storage class is applied via the x-amz-storage-class header for
both regular PUT uploads and multipart upload initiation.
This commit is contained in:
Bernardo Meurer Costa 2025-10-23 04:32:58 +00:00
parent a786c9eedb
commit 4e64dea21b
No known key found for this signature in database
4 changed files with 65 additions and 2 deletions

View file

@ -0,0 +1,21 @@
---
synopsis: "S3 binary cache stores now support storage class configuration"
prs: [14464]
issues: [7015]
---
S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
Example usage:
```bash
# Use Glacier storage for long-term archival
nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
# Use Intelligent Tiering for automatic cost optimization
nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
```
The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.

View file

@ -122,4 +122,22 @@ TEST(S3BinaryCacheStore, parameterFiltering)
EXPECT_EQ(ref.params["priority"], "10"); EXPECT_EQ(ref.params["priority"], "10");
} }
/**
* Test storage class configuration
*/
TEST(S3BinaryCacheStore, storageClassDefault)
{
S3BinaryCacheStoreConfig config{"s3", "test-bucket", {}};
EXPECT_EQ(config.storageClass.get(), std::nullopt);
}
TEST(S3BinaryCacheStore, storageClassConfiguration)
{
StringMap params;
params["storage-class"] = "GLACIER";
S3BinaryCacheStoreConfig config("s3", "test-bucket", params);
EXPECT_EQ(config.storageClass.get(), std::optional<std::string>("GLACIER"));
}
} // namespace nix } // namespace nix

View file

@ -93,6 +93,26 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
Default is 100 MiB. Only takes effect when multipart-upload is enabled. Default is 100 MiB. Only takes effect when multipart-upload is enabled.
)"}; )"};
const Setting<std::optional<std::string>> storageClass{
this,
std::nullopt,
"storage-class",
R"(
The S3 storage class to use for uploaded objects. When not set (default),
uses the bucket's default storage class. Valid values include:
- STANDARD (default, frequently accessed data)
- REDUCED_REDUNDANCY (less frequently accessed data)
- STANDARD_IA (infrequent access)
- ONEZONE_IA (infrequent access, single AZ)
- INTELLIGENT_TIERING (automatic cost optimization)
- GLACIER (archival with retrieval times in minutes to hours)
- DEEP_ARCHIVE (long-term archival with 12-hour retrieval)
- GLACIER_IR (instant retrieval archival)
See AWS S3 documentation for detailed storage class descriptions and pricing:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html
)"};
/** /**
* Set of settings that are part of the S3 URI itself. * Set of settings that are part of the S3 URI itself.
* These are needed for region specification and other S3-specific settings. * These are needed for region specification and other S3-specific settings.

View file

@ -134,10 +134,14 @@ void S3BinaryCacheStore::upsertFile(
const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint)
{ {
auto doUpload = [&](RestartableSource & src, uint64_t size, std::optional<Headers> headers) { auto doUpload = [&](RestartableSource & src, uint64_t size, std::optional<Headers> headers) {
Headers uploadHeaders = headers.value_or(Headers());
if (auto storageClass = s3Config->storageClass.get()) {
uploadHeaders.emplace_back("x-amz-storage-class", *storageClass);
}
if (s3Config->multipartUpload && size > s3Config->multipartThreshold) { if (s3Config->multipartUpload && size > s3Config->multipartThreshold) {
uploadMultipart(path, src, size, mimeType, std::move(headers)); uploadMultipart(path, src, size, mimeType, std::move(uploadHeaders));
} else { } else {
upload(path, src, size, mimeType, std::move(headers)); upload(path, src, size, mimeType, std::move(uploadHeaders));
} }
}; };