nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-14 22:42:41 +01:00
Code Activity

feat(libstore/s3): add toHttpsUrl

Browse source 
This is extracted from the work in #13752
This commit is contained in:
Bernardo Meurer Costa 2025-08-25 17:09:07 +00:00
parent 9bee0fa6ac
commit 5985d67906
No known key found for this signature in database
3 changed files with 149 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

110
src/libstore-tests/s3.cc
View file

@ -35,7 +35,8 @@ INSTANTIATE_TEST_SUITE_P(
.bucket = "my-bucket",
.key = "my-key.txt",
},
"basic_s3_bucket"},
"basic_s3_bucket",
},
ParsedS3URLTestCase{
"s3://prod-cache/nix/store/abc123.nar.xz?region=eu-west-1",
{
@ -43,7 +44,8 @@ INSTANTIATE_TEST_SUITE_P(
.key = "nix/store/abc123.nar.xz",
.region = "eu-west-1",
},
"with_region"},
"with_region",
},
ParsedS3URLTestCase{
"s3://bucket/key?region=us-west-2&profile=prod&endpoint=custom.s3.com&scheme=https&region=us-east-1",
{
@ -54,7 +56,8 @@ INSTANTIATE_TEST_SUITE_P(
.scheme = "https",
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
},
"complex"},
"complex",
},
ParsedS3URLTestCase{
"s3://cache/file.txt?profile=production&region=ap-southeast-2",
{
@ -63,7 +66,8 @@ INSTANTIATE_TEST_SUITE_P(
.profile = "production",
.region = "ap-southeast-2",
},
"with_profile_and_region"},
"with_profile_and_region",
},
ParsedS3URLTestCase{
"s3://bucket/key?endpoint=https://minio.local&scheme=http",
{
@ -77,7 +81,8 @@ INSTANTIATE_TEST_SUITE_P(
.authority = ParsedURL::Authority{.host = "minio.local"},
},
},
"with_absolute_endpoint_uri"}),
"with_absolute_endpoint_uri",
}),
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
TEST(InvalidParsedS3URLTest, parseS3URLErrors)
@ -91,6 +96,101 @@ TEST(InvalidParsedS3URLTest, parseS3URLErrors)
ASSERT_THAT([]() { ParsedS3URL::parse(parseURL("s3://127.0.0.1")); }, invalidBucketMatcher);
}
// Parameterized test for s3ToHttpsUrl conversion
struct S3ToHttpsConversionTestCase
{
ParsedS3URL input;
ParsedURL expected;
std::string description;
};
class S3ToHttpsConversionTest : public ::testing::WithParamInterface<S3ToHttpsConversionTestCase>,
public ::testing::Test
{};
TEST_P(S3ToHttpsConversionTest, ConvertsCorrectly)
{
const auto & testCase = GetParam();
auto result = testCase.input.toHttpsUrl();
EXPECT_EQ(result, testCase.expected) << "Failed for: " << testCase.description;
}
INSTANTIATE_TEST_SUITE_P(
S3ToHttpsConversion,
S3ToHttpsConversionTest,
::testing::Values(
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
.key = "my-key.txt",
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = "/my-bucket/my-key.txt",
},
"basic_s3_default_region",
},
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "prod-cache",
.key = "nix/store/abc123.nar.xz",
.region = "eu-west-1",
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = "/prod-cache/nix/store/abc123.nar.xz",
},
"with_eu_west_1_region",
},
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = "key",
.scheme = "http",
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "custom.s3.com"},
.path = "/bucket/key",
},
"custom_endpoint_authority",
},
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = "key",
.endpoint =
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
},
},
ParsedURL{
.scheme = "http",
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
.path = "/bucket/key",
},
"custom_endpoint_with_port",
},
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "bucket",
.key = "path/to/file.txt",
.region = "ap-southeast-2",
.scheme = "https",
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
.path = "/bucket/path/to/file.txt",
},
"complex_path_and_region",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
} // namespace nix
#endif

6
src/libstore/include/nix/store/s3.hh
View file

@ -75,6 +75,12 @@ struct ParsedS3URL
}
static ParsedS3URL parse(const ParsedURL & uri);
/**
* Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication
*/
ParsedURL toHttpsUrl() const;
auto operator<=>(const ParsedS3URL & other) const = default;
};

38
src/libstore/s3.cc
View file

@ -1,6 +1,8 @@
#include "nix/store/s3.hh"
#include "nix/util/split.hh"
#include "nix/util/url.hh"
#include "nix/util/util.hh"
#include "nix/util/canon-path.hh"
namespace nix {
@ -64,6 +66,42 @@ try {
throw;
}
ParsedURL ParsedS3URL::toHttpsUrl() const
{
std::string regionStr = region.value_or("us-east-1");
std::string schemeStr = scheme.value_or("https");
// Handle endpoint configuration using std::visit
return std::visit(
overloaded{
[&](const std::monostate &) {
// No custom endpoint, use standard AWS S3 endpoint
return ParsedURL{
.scheme = schemeStr,
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
.path = (CanonPath::root / bucket / CanonPath(key)).abs(),
};
},
[&](const ParsedURL::Authority & auth) {
// Endpoint is just an authority (hostname/port)
return ParsedURL{
.scheme = schemeStr,
.authority = auth,
.path = (CanonPath::root / bucket / CanonPath(key)).abs(),
};
},
[&](const ParsedURL & endpointUrl) {
// Endpoint is already a ParsedURL (e.g., http://server:9000)
return ParsedURL{
.scheme = endpointUrl.scheme,
.authority = endpointUrl.authority,
.path = (CanonPath(endpointUrl.path) / bucket / CanonPath(key)).abs(),
};
},
},
endpoint);
}
#endif
} // namespace nix