From d18f959d4fb381ec4e3a489410fb336731cff7d3 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sun, 12 Oct 2025 02:07:04 +0000 Subject: [PATCH] test(nixos): add comprehensive curl-based S3 VM tests Add `curl-s3-binary-cache-store.nix` with comprehensive test coverage for the curl-based S3 implementation. Depends-On: #14206, #14222 --- ci/gha/tests/default.nix | 4 + tests/nixos/curl-s3-binary-cache-store.nix | 507 +++++++++++++++++++++ tests/nixos/default.nix | 2 + 3 files changed, 513 insertions(+) create mode 100644 tests/nixos/curl-s3-binary-cache-store.nix diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 09fb6ec23..46310bc36 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -236,6 +236,10 @@ rec { # S3 binary cache store test only runs when S3 support is enabled inherit (nixosTests) s3-binary-cache-store; } + // lib.optionalAttrs (withCurlS3 == true) { + # S3 binary cache store test using curl implementation + inherit (nixosTests) curl-s3-binary-cache-store; + } // lib.optionalAttrs (!withSanitizers && !withCoverage) { # evalNixpkgs uses non-instrumented components from hydraJobs, so only run it # when not testing with sanitizers to avoid rebuilding nix diff --git a/tests/nixos/curl-s3-binary-cache-store.nix b/tests/nixos/curl-s3-binary-cache-store.nix new file mode 100644 index 000000000..53d79689c --- /dev/null +++ b/tests/nixos/curl-s3-binary-cache-store.nix @@ -0,0 +1,507 @@ +{ + lib, + config, + nixpkgs, + ... +}: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Test packages - minimal packages for fast copying + pkgA = pkgs.writeText "test-package-a" "test package a"; + pkgB = pkgs.writeText "test-package-b" "test package b"; + pkgC = pkgs.writeText "test-package-c" "test package c"; + + # S3 configuration + accessKey = "BKIKJAA5BMMU2RHO6IBB"; + secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12"; + +in +{ + name = "curl-s3-binary-cache-store"; + + nodes = { + server = + { + config, + lib, + pkgs, + ... + }: + { + virtualisation.writableStore = true; + virtualisation.cores = 2; + virtualisation.additionalPaths = [ + pkgA + pkgB + pkgC + ]; + environment.systemPackages = [ pkgs.minio-client ]; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; + services.minio = { + enable = true; + region = "eu-west-1"; + rootCredentialsFile = pkgs.writeText "minio-credentials-full" '' + MINIO_ROOT_USER=${accessKey} + MINIO_ROOT_PASSWORD=${secretKey} + ''; + }; + networking.firewall.allowedTCPPorts = [ 9000 ]; + }; + + client = + { config, pkgs, ... }: + { + virtualisation.writableStore = true; + virtualisation.cores = 2; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; + }; + }; + + testScript = + { nodes }: + # python + '' + import json + import random + import re + import uuid + + # ============================================================================ + # Configuration + # ============================================================================ + + ACCESS_KEY = '${accessKey}' + SECRET_KEY = '${secretKey}' + ENDPOINT = 'http://server:9000' + REGION = 'eu-west-1' + + PKG_A = '${pkgA}' + PKG_B = '${pkgB}' + PKG_C = '${pkgC}' + + ENV_WITH_CREDS = f"AWS_ACCESS_KEY_ID={ACCESS_KEY} AWS_SECRET_ACCESS_KEY={SECRET_KEY}" + + # ============================================================================ + # Helper Functions + # ============================================================================ + + def make_s3_url(bucket, path="", **params): + """Build S3 URL with optional path and query parameters""" + params.setdefault('endpoint', ENDPOINT) + params.setdefault('region', REGION) + query = '&'.join(f"{k}={v}" for k, v in params.items()) + bucket_and_path = f"{bucket}{path}" if path else bucket + return f"s3://{bucket_and_path}?{query}" + + def make_http_url(path): + """Build HTTP URL for direct S3 access""" + return f"{ENDPOINT}/{path}" + + def get_package_hash(pkg_path): + """Extract store hash from package path""" + return pkg_path.split("/")[-1].split("-")[0] + + def verify_content_encoding(machine, bucket, object_path, expected_encoding): + """Verify S3 object has expected Content-Encoding header""" + stat = machine.succeed(f"mc stat minio/{bucket}/{object_path}") + if "Content-Encoding" not in stat or expected_encoding not in stat: + print(f"mc stat output for {object_path}:") + print(stat) + raise Exception(f"Expected Content-Encoding: {expected_encoding} header on {object_path}") + + def verify_no_compression(machine, bucket, object_path): + """Verify S3 object has no compression headers""" + stat = machine.succeed(f"mc stat minio/{bucket}/{object_path}") + if "Content-Encoding" in stat and ("gzip" in stat or "xz" in stat): + print(f"mc stat output for {object_path}:") + print(stat) + raise Exception(f"Object {object_path} should not have compression Content-Encoding") + + def assert_count(output, pattern, expected, error_msg): + """Assert that pattern appears exactly expected times in output""" + actual = output.count(pattern) + if actual != expected: + print("Debug output:") + print(output) + raise Exception(f"{error_msg}: expected {expected}, got {actual}") + + def with_test_bucket(populate_with=[]): + """ + Decorator that creates/destroys a unique bucket for each test. + Optionally pre-populates bucket with specified packages. + + Args: + populate_with: List of packages to upload before test runs + """ + def decorator(test_func): + def wrapper(): + bucket = str(uuid.uuid4()) + server.succeed(f"mc mb minio/{bucket}") + try: + if populate_with: + store_url = make_s3_url(bucket) + for pkg in populate_with: + server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {pkg}") + test_func(bucket) + finally: + server.succeed(f"mc rb --force minio/{bucket}") + return wrapper + return decorator + + # ============================================================================ + # Test Functions + # ============================================================================ + + @with_test_bucket() + def test_credential_caching(bucket): + """Verify credential providers are cached and reused""" + print("\n=== Testing Credential Caching ===") + + store_url = make_s3_url(bucket) + output = server.succeed( + f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' " + f"{PKG_A} {PKG_B} {PKG_C} 2>&1" + ) + + assert_count( + output, + "creating new AWS credential provider", + 1, + "Credential provider caching failed" + ) + + print("✓ Credential provider created once and cached") + + @with_test_bucket(populate_with=[PKG_A]) + def test_fetchurl_basic(bucket): + """Test builtins.fetchurl works with s3:// URLs""" + print("\n=== Testing builtins.fetchurl ===") + + client.wait_for_unit("network-addresses-eth1.service") + + cache_info_url = make_s3_url(bucket, path="/nix-cache-info") + + client.succeed( + f"{ENV_WITH_CREDS} nix eval --impure --expr " + f"'builtins.fetchurl {{ name = \"foo\"; url = \"{cache_info_url}\"; }}'" + ) + + print("✓ builtins.fetchurl works with s3:// URLs") + + @with_test_bucket() + def test_error_message_formatting(bucket): + """Verify error messages display URLs correctly""" + print("\n=== Testing Error Message Formatting ===") + + nonexistent_url = make_s3_url(bucket, path="/foo-that-does-not-exist") + expected_http_url = make_http_url(f"{bucket}/foo-that-does-not-exist") + + error_msg = client.fail( + f"{ENV_WITH_CREDS} nix eval --impure --expr " + f"'builtins.fetchurl {{ name = \"foo\"; url = \"{nonexistent_url}\"; }}' 2>&1" + ) + + if f"unable to download '{expected_http_url}': HTTP error 404" not in error_msg: + print("Actual error message:") + print(error_msg) + raise Exception("Error message formatting failed - should show actual URL, not %s placeholder") + + print("✓ Error messages format URLs correctly") + + @with_test_bucket(populate_with=[PKG_A]) + def test_fork_credential_preresolution(bucket): + """Test credential pre-resolution in forked processes""" + print("\n=== Testing Fork Credential Pre-resolution ===") + + # Get hash of nix-cache-info for fixed-output derivation + cache_info_url = make_s3_url(bucket, path="/nix-cache-info") + + cache_info_path = client.succeed( + f"{ENV_WITH_CREDS} nix eval --impure --raw --expr " + f"'builtins.fetchurl {{ name = \"nix-cache-info\"; url = \"{cache_info_url}\"; }}'" + ).strip() + + cache_info_hash = client.succeed( + f"nix hash file --type sha256 --base32 {cache_info_path}" + ).strip() + + # Build derivation with unique test ID + test_id = random.randint(0, 10000) + test_url = make_s3_url(bucket, path="/nix-cache-info", test_id=test_id) + + fetchurl_expr = """ + import {{ + name = "s3-fork-test-{id}"; + url = "{url}"; + sha256 = "{hash}"; + }} + """.format(id=test_id, url=test_url, hash=cache_info_hash) + + output = client.succeed( + f"{ENV_WITH_CREDS} nix build --debug --impure --expr '{fetchurl_expr}' 2>&1" + ) + + # Verify fork behavior + if "builtin:fetchurl creating fresh FileTransfer instance" not in output: + print("Debug output:") + print(output) + raise Exception("Expected to find FileTransfer creation in forked process") + + print(" ✓ Forked process creates fresh FileTransfer") + + # Verify pre-resolution in parent + required_messages = [ + "Pre-resolving AWS credentials for S3 URL in builtin:fetchurl", + "Successfully pre-resolved AWS credentials in parent process", + ] + + for msg in required_messages: + if msg not in output: + print("Debug output:") + print(output) + raise Exception(f"Missing expected message: {msg}") + + print(" ✓ Parent pre-resolves credentials") + + # Verify child uses pre-resolved credentials + if "Using pre-resolved AWS credentials from parent process" not in output: + print("Debug output:") + print(output) + raise Exception("Child should use pre-resolved credentials") + + # Extract child PID and verify it doesn't create new providers + filetransfer_match = re.search( + r'\[pid=(\d+)\] builtin:fetchurl creating fresh FileTransfer instance', + output + ) + + if not filetransfer_match: + raise Exception("Could not extract child PID from debug output") + + child_pid = filetransfer_match.group(1) + child_provider_creation = f"[pid={child_pid}] creating new AWS credential provider" + + if child_provider_creation in output: + print("Debug output:") + print(output) + raise Exception(f"Child process (pid={child_pid}) should NOT create new credential providers") + + print(" ✓ Child uses pre-resolved credentials (no new providers)") + + @with_test_bucket(populate_with=[PKG_A, PKG_B, PKG_C]) + def test_store_operations(bucket): + """Test nix store info and copy operations""" + print("\n=== Testing Store Operations ===") + + store_url = make_s3_url(bucket) + + # Verify store info works + client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{store_url}' >&2") + + # Get and validate store info JSON + info_json = client.succeed(f"{ENV_WITH_CREDS} nix store info --json --store '{store_url}'") + store_info = json.loads(info_json) + + if not store_info.get("url"): + raise Exception("Store should have a URL") + + print(f" ✓ Store URL: {store_info['url']}") + + # Test copy from store + client.fail(f"nix path-info {PKG_A}") + + output = client.succeed( + f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs " + f"--from '{store_url}' {PKG_A} {PKG_B} {PKG_C} 2>&1" + ) + + assert_count( + output, + "creating new AWS credential provider", + 1, + "Client credential provider caching failed" + ) + + client.succeed(f"nix path-info {PKG_A}") + + print(" ✓ nix copy works") + print(" ✓ Credentials cached on client") + + @with_test_bucket(populate_with=[PKG_A]) + def test_url_format_variations(bucket): + """Test different S3 URL parameter combinations""" + print("\n=== Testing URL Format Variations ===") + + # Test parameter order variation (region before endpoint) + url1 = f"s3://{bucket}?region={REGION}&endpoint={ENDPOINT}" + client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url1}' >&2") + print(" ✓ Parameter order: region before endpoint works") + + # Test parameter order variation (endpoint before region) + url2 = f"s3://{bucket}?endpoint={ENDPOINT}®ion={REGION}" + client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2") + print(" ✓ Parameter order: endpoint before region works") + + @with_test_bucket(populate_with=[PKG_A]) + def test_concurrent_fetches(bucket): + """Validate thread safety with concurrent S3 operations""" + print("\n=== Testing Concurrent Fetches ===") + + # Get hash for test derivations + cache_info_url = make_s3_url(bucket, path="/nix-cache-info") + + cache_info_path = client.succeed( + f"{ENV_WITH_CREDS} nix eval --impure --raw --expr " + f"'builtins.fetchurl {{ name = \"nix-cache-info\"; url = \"{cache_info_url}\"; }}'" + ).strip() + + cache_info_hash = client.succeed( + f"nix hash file --type sha256 --base32 {cache_info_path}" + ).strip() + + # Create 5 concurrent fetch derivations + # Build base URL for concurrent test (we'll add fetch_id in Nix interpolation) + base_url = make_s3_url(bucket, path="/nix-cache-info") + concurrent_expr = """ + let + mkFetch = i: import {{ + name = "concurrent-s3-fetch-''${{toString i}}"; + url = "{url}&fetch_id=''${{toString i}}"; + sha256 = "{hash}"; + }}; + fetches = builtins.listToAttrs (map (i: {{ + name = "fetch''${{toString i}}"; + value = mkFetch i; + }}) (builtins.genList (i: i) 5)); + in fetches + """.format(url=base_url, hash=cache_info_hash) + + try: + output = client.succeed( + f"{ENV_WITH_CREDS} nix build --debug --impure " + f"--expr '{concurrent_expr}' --max-jobs 5 2>&1" + ) + except: + output = client.fail( + f"{ENV_WITH_CREDS} nix build --debug --impure " + f"--expr '{concurrent_expr}' --max-jobs 5 2>&1" + ) + + if "error:" in output.lower(): + print("Found error during concurrent fetches:") + print(output) + + providers_created = output.count("creating new AWS credential provider") + transfers_created = output.count("builtin:fetchurl creating fresh FileTransfer instance") + + print(f" ✓ {providers_created} credential providers created") + print(f" ✓ {transfers_created} FileTransfer instances created") + + if transfers_created != 5: + print("Debug output:") + print(output) + raise Exception( + f"Expected 5 FileTransfer instances for 5 concurrent fetches, got {transfers_created}" + ) + + @with_test_bucket() + def test_compression_narinfo_gzip(bucket): + """Test narinfo compression with gzip""" + print("\n=== Testing Compression: narinfo (gzip) ===") + + store_url = make_s3_url(bucket, **{'narinfo-compression': 'gzip'}) + server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_B}") + + pkg_hash = get_package_hash(PKG_B) + verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "gzip") + + print(" ✓ .narinfo has Content-Encoding: gzip") + + # Verify client can download and decompress + client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_B}") + client.succeed(f"nix path-info {PKG_B}") + + print(" ✓ Client decompressed .narinfo successfully") + + @with_test_bucket() + def test_compression_mixed(bucket): + """Test mixed compression (narinfo=xz, ls=gzip)""" + print("\n=== Testing Compression: mixed (narinfo=xz, ls=gzip) ===") + + store_url = make_s3_url( + bucket, + **{'narinfo-compression': 'xz', 'write-nar-listing': 'true', 'ls-compression': 'gzip'} + ) + + server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_C}") + + pkg_hash = get_package_hash(PKG_C) + + # Verify .narinfo has xz compression + verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz") + print(" ✓ .narinfo has Content-Encoding: xz") + + # Verify .ls has gzip compression + verify_content_encoding(server, bucket, f"{pkg_hash}.ls", "gzip") + print(" ✓ .ls has Content-Encoding: gzip") + + # Verify client can download with mixed compression + client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_C}") + client.succeed(f"nix path-info {PKG_C}") + + print(" ✓ Client downloaded package with mixed compression") + + @with_test_bucket() + def test_compression_disabled(bucket): + """Verify no compression by default""" + print("\n=== Testing Compression: disabled (default) ===") + + store_url = make_s3_url(bucket) + server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_A}") + + pkg_hash = get_package_hash(PKG_A) + verify_no_compression(server, bucket, f"{pkg_hash}.narinfo") + + print(" ✓ No compression applied by default") + + # ============================================================================ + # Main Test Execution + # ============================================================================ + + print("\n" + "="*80) + print("S3 Binary Cache Store Tests") + print("="*80) + + start_all() + + # Initialize MinIO server + server.wait_for_unit("minio") + server.wait_for_unit("network-addresses-eth1.service") + server.wait_for_open_port(9000) + server.succeed(f"mc config host add minio http://localhost:9000 {ACCESS_KEY} {SECRET_KEY} --api s3v4") + + # Run tests (each gets isolated bucket via decorator) + test_credential_caching() + test_fetchurl_basic() + test_error_message_formatting() + test_fork_credential_preresolution() + test_store_operations() + test_url_format_variations() + test_concurrent_fetches() + test_compression_narinfo_gzip() + test_compression_mixed() + test_compression_disabled() + + print("\n" + "="*80) + print("✓ All S3 Binary Cache Store Tests Passed!") + print("="*80) + ''; +} diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index edfa4124f..ea6a7e914 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -201,6 +201,8 @@ in s3-binary-cache-store = runNixOSTest ./s3-binary-cache-store.nix; + curl-s3-binary-cache-store = runNixOSTest ./curl-s3-binary-cache-store.nix; + fsync = runNixOSTest ./fsync.nix; cgroups = runNixOSTest ./cgroups;