nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-08 19:46:02 +01:00
Code Activity

refactor(libstore): withCurlS3 -> withAWS

Browse source 
Now that the legacy S3 implementation is gone, we can go back to calling
things `NIX_WITH_S3_SUPPORT`.
This commit is contained in:
Bernardo Meurer Costa 2025-10-14 23:58:48 +00:00
parent 9295c14a35
commit 1f710300c9
No known key found for this signature in database
19 changed files with 42 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

14
.github/workflows/ci.yml vendored
View file

@ -67,7 +67,7 @@ jobs:
instrumented: false
primary: true
stdenv: stdenv
withCurlS3: true
withAWS: true
# TODO: remove once curl-based-s3 fully lands
- scenario: on ubuntu (no s3)
runs-on: ubuntu-24.04
@ -75,21 +75,21 @@ jobs:
instrumented: false
primary: false
stdenv: stdenv
withCurlS3: false
withAWS: false
- scenario: on macos
runs-on: macos-14
os: darwin
instrumented: false
primary: true
stdenv: stdenv
withCurlS3: true
withAWS: true
- scenario: on ubuntu (with sanitizers / coverage)
runs-on: ubuntu-24.04
os: linux
instrumented: true
primary: false
stdenv: clangStdenv
withCurlS3: true
withAWS: true
name: tests ${{ matrix.scenario }}
runs-on: ${{ matrix.runs-on }}
timeout-minutes: 60
@ -113,13 +113,13 @@ jobs:
nix build --file ci/gha/tests/wrapper.nix componentTests -L \
--arg withInstrumentation ${{ matrix.instrumented }} \
--argstr stdenv "${{ matrix.stdenv }}" \
${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
${{ format('--arg withAWS {0}', matrix.withAWS) }}
- name: Run VM tests
run: |
nix build --file ci/gha/tests/wrapper.nix vmTests -L \
--arg withInstrumentation ${{ matrix.instrumented }} \
--argstr stdenv "${{ matrix.stdenv }}" \
${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
${{ format('--arg withAWS {0}', matrix.withAWS) }}
if: ${{ matrix.os == 'linux' }}
- name: Run flake checks and prepare the installer tarball
run: |
@ -131,7 +131,7 @@ jobs:
nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \
--arg withInstrumentation ${{ matrix.instrumented }} \
--argstr stdenv "${{ matrix.stdenv }}" \
${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }} \
${{ format('--arg withAWS {0}', matrix.withAWS) }} \
--out-link coverage-reports
cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
if: ${{ matrix.instrumented }}

8
ci/gha/tests/default.nix
View file

@ -12,7 +12,7 @@
componentTestsPrefix ? "",
withSanitizers ? false,
withCoverage ? false,
withCurlS3 ? null,
withAWS ? null,
...
}:
@ -58,9 +58,7 @@ rec {
nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; };
# Override AWS configuration if specified
nix-store = prev.nix-store.override (
lib.optionalAttrs (withCurlS3 != null) { inherit withCurlS3; }
);
nix-store = prev.nix-store.override (lib.optionalAttrs (withAWS != null) { inherit withAWS; });
mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
# Unclear how to make Perl bindings work with a dynamically linked ASAN.
@ -229,7 +227,7 @@ rec {
vmTests = {
}
// lib.optionalAttrs (withCurlS3 == true) {
// lib.optionalAttrs (withAWS == true) {
# S3 binary cache store test using curl implementation
inherit (nixosTests) curl-s3-binary-cache-store;
}

4
ci/gha/tests/wrapper.nix
View file

@ -5,7 +5,7 @@
stdenv ? "stdenv",
componentTestsPrefix ? "",
withInstrumentation ? false,
withCurlS3 ? null,
withAWS ? null,
}@args:
import ./. (
args
@ -13,6 +13,6 @@ import ./. (
getStdenv = p: p.${stdenv};
withSanitizers = withInstrumentation;
withCoverage = withInstrumentation;
inherit withCurlS3;
inherit withAWS;
}
)

2
src/libstore-tests/s3-binary-cache-store.cc
View file

@ -1,6 +1,6 @@
#include "nix/store/s3-binary-cache-store.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/store/http-binary-cache-store.hh"
# include "nix/store/filetransfer.hh"

2
src/libstore-tests/s3-url.cc
View file

@ -1,7 +1,7 @@
#include "nix/store/s3-url.hh"
#include "nix/util/tests/gmock-matchers.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include <gtest/gtest.h>
# include <gmock/gmock.h>

2
src/libstore/aws-creds.cc
View file

@ -1,6 +1,6 @@
#include "nix/store/aws-creds.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include <aws/crt/Types.h>
# include "nix/store/s3-url.hh"

2
src/libstore/builtins/fetchurl.cc
View file

@ -41,7 +41,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
FileTransferRequest request(VerbatimURL{url});
request.decompress = false;
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
// Use pre-resolved credentials if available
if (ctx.awsCredentials && request.uri.scheme() == "s3") {
debug("[pid=%d] Using pre-resolved AWS credentials from parent process", getpid());

10
src/libstore/filetransfer.cc
View file

@ -9,7 +9,7 @@
#include "store-config-private.hh"
#include <optional>
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/store/aws-creds.hh"
# include "nix/store/s3-url.hh"
#endif
@ -435,7 +435,7 @@ struct curlFileTransfer : public FileTransfer
}
}
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
// Set up AWS SigV4 signing if this is an S3 request
// Note: AWS SigV4 support guaranteed available (curl >= 7.75.0 checked at build time)
// The username/password (access key ID and secret key) are set via the general
@ -820,7 +820,7 @@ struct curlFileTransfer : public FileTransfer
void enqueueItem(std::shared_ptr<TransferItem> item)
{
if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
&& item->request.uri.scheme() != "s3"
#endif
)
@ -841,7 +841,7 @@ struct curlFileTransfer : public FileTransfer
{
/* Ugly hack to support s3:// URIs. */
if (request.uri.scheme() == "s3") {
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
// New curl-based S3 implementation
auto modifiedRequest = request;
modifiedRequest.setupForS3();
@ -876,7 +876,7 @@ ref<FileTransfer> makeFileTransfer()
return makeCurlFileTransfer();
}
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
void FileTransferRequest::setupForS3()
{
auto parsedS3 = ParsedS3URL::parse(uri.parsed());

2
src/libstore/include/nix/store/aws-creds.hh
View file

@ -2,7 +2,7 @@
///@file
#include "nix/store/config.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/store/s3-url.hh"
# include "nix/util/error.hh"

4
src/libstore/include/nix/store/builtins.hh
View file

@ -4,7 +4,7 @@
#include "nix/store/derivations.hh"
#include "nix/store/config.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/store/aws-creds.hh"
#endif
@ -18,7 +18,7 @@ struct BuiltinBuilderContext
std::string caFileData;
Path tmpDirInSandbox;
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
/**
* Pre-resolved AWS credentials for S3 URLs in builtin:fetchurl.
* When present, these should be used instead of creating new credential providers.

6
src/libstore/include/nix/store/filetransfer.hh
View file

@ -12,7 +12,7 @@
#include "nix/util/url.hh"
#include "nix/store/config.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/store/aws-creds.hh"
#endif
@ -113,7 +113,7 @@ struct FileTransferRequest
* When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option.
*/
std::optional<UsernameAuth> usernameAuth;
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
/**
* Pre-resolved AWS session token for S3 requests.
* When provided along with usernameAuth, this will be used instead of fetching fresh credentials.
@ -132,7 +132,7 @@ struct FileTransferRequest
return data ? "upload" : "download";
}
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
private:
friend struct curlFileTransfer;
void setupForS3();

2
src/libstore/include/nix/store/s3-binary-cache-store.hh
View file

@ -3,7 +3,7 @@
#include "nix/store/config.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/store/http-binary-cache-store.hh"

2
src/libstore/include/nix/store/s3-url.hh
View file

@ -2,7 +2,7 @@
///@file
#include "nix/store/config.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/util/url.hh"
# include "nix/util/util.hh"

2
src/libstore/meson.build
View file

@ -160,7 +160,7 @@ if curl_s3_store_opt.enabled()
deps_other += aws_crt_cpp
endif
configdata_pub.set('NIX_WITH_CURL_S3', curl_s3_store_opt.enabled().to_int())
configdata_pub.set('NIX_WITH_S3_SUPPORT', curl_s3_store_opt.enabled().to_int())
subdir('nix-meson-build-support/generate-header')

6
src/libstore/package.nix
View file

@ -23,7 +23,7 @@
embeddedSandboxShell ? stdenv.hostPlatform.isStatic,
withCurlS3 ?
withAWS ?
# Default is this way because there have been issues building this dependency
stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin),
}:
@ -65,7 +65,7 @@ mkMesonLibrary (finalAttrs: {
sqlite
]
++ lib.optional stdenv.hostPlatform.isLinux libseccomp
++ lib.optional withCurlS3 aws-crt-cpp;
++ lib.optional withAWS aws-crt-cpp;
propagatedBuildInputs = [
nix-util
@ -75,7 +75,7 @@ mkMesonLibrary (finalAttrs: {
mesonFlags = [
(lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)
(lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell)
(lib.mesonEnable "curl-s3-store" withCurlS3)
(lib.mesonEnable "curl-s3-store" withAWS)
]
++ lib.optionals stdenv.hostPlatform.isLinux [
(lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")

2
src/libstore/s3-binary-cache-store.cc
View file

@ -1,6 +1,6 @@
#include "nix/store/s3-binary-cache-store.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include <cassert>

2
src/libstore/s3-url.cc
View file

@ -1,6 +1,6 @@
#include "nix/store/s3-url.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/util/error.hh"
# include "nix/util/split.hh"

12
src/libstore/unix/build/derivation-builder.cc
View file

@ -46,7 +46,7 @@
#include "store-config-private.hh"
#include "build/derivation-check.hh"
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
# include "nix/store/aws-creds.hh"
# include "nix/store/s3-url.hh"
# include "nix/util/url.hh"
@ -296,7 +296,7 @@ protected:
*/
virtual void startChild();
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
/**
* Pre-resolve AWS credentials for S3 URLs in builtin:fetchurl.
* This should be called before forking to ensure credentials are available in child.
@ -359,7 +359,7 @@ protected:
*/
struct RunChildArgs
{
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
std::optional<AwsCredentials> awsCredentials;
#endif
};
@ -945,7 +945,7 @@ void DerivationBuilderImpl::openSlave()
throw SysError("cannot pipe standard error into log file");
}
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
std::optional<AwsCredentials> DerivationBuilderImpl::preResolveAwsCredentials()
{
if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") {
@ -974,7 +974,7 @@ std::optional<AwsCredentials> DerivationBuilderImpl::preResolveAwsCredentials()
void DerivationBuilderImpl::startChild()
{
RunChildArgs args{
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
.awsCredentials = preResolveAwsCredentials(),
#endif
};
@ -1255,7 +1255,7 @@ void DerivationBuilderImpl::runChild(RunChildArgs args)
BuiltinBuilderContext ctx{
.drv = drv,
.tmpDirInSandbox = tmpDirInSandbox(),
#if NIX_WITH_CURL_S3
#if NIX_WITH_S3_SUPPORT
.awsCredentials = args.awsCredentials,
#endif
};

2
src/libstore/unix/build/linux-derivation-builder.cc
View file

@ -277,7 +277,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
void startChild() override
{
RunChildArgs args{
# if NIX_WITH_CURL_S3
# if NIX_WITH_S3_SUPPORT
.awsCredentials = preResolveAwsCredentials(),
# endif
};