Merge pull request #11691 from NixOS/mergify/bp/2.24-maintenance/pr-11677

builtins.fetchurl: Fix segfault on s3:// URLs (backport #11677)

src/libfetchers/tarball.cc

@@ -90,6 +90,7 @@ DownloadFileResult downloadFile(
     /* Cache metadata for all URLs in the redirect chain. */
     for (auto & url : res.urls) {
         key.second.insert_or_assign("url", url);
+        assert(!res.urls.empty());
         infoAttrs.insert_or_assign("url", *res.urls.rbegin());
         getCache()->upsert(key, *store, infoAttrs, *storePath);
     }

src/libstore/filetransfer.cc

@@ -754,12 +754,17 @@ struct curlFileTransfer : public FileTransfer
 
                 S3Helper s3Helper(profile, region, scheme, endpoint);
 
+                Activity act(*logger, lvlTalkative, actFileTransfer,
+                    fmt("downloading '%s'", request.uri),
+                    {request.uri}, request.parentAct);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;
                 if (!s3Res.data)
                     throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri);
                 res.data = std::move(*s3Res.data);
+                res.urls.push_back(request.uri);
                 callback(std::move(res));
 #else
                 throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);

src/libstore/s3-binary-cache-store.cc

@@ -9,6 +9,7 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
+#include "signals.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -117,6 +118,7 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
 {
     bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
     {
+        checkInterrupt();
         auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
         if (retry)
             printError("AWS error '%s' (%s), will retry in %d ms",

tests/nixos/default.nix

@@ -148,4 +148,6 @@ in
   user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing;
 
   fetchurl = runNixOSTestFor "x86_64-linux" ./fetchurl.nix;
+
+  s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix;
 }

tests/nixos/nix-copy-closure.nix

@@ -1,6 +1,6 @@
 # Test ‘nix-copy-closure’.
 
-{ lib, config, nixpkgs, hostPkgs, ... }:
+{ lib, config, nixpkgs, ... }:
 
 let
   pkgs = config.nodes.client.nixpkgs.pkgs;

tests/nixos/s3-binary-cache-store.nix

@@ -0,0 +1,66 @@
+{ lib, config, nixpkgs, ... }:
+
+let
+  pkgs = config.nodes.client.nixpkgs.pkgs;
+
+  pkgA = pkgs.cowsay;
+
+  accessKey = "BKIKJAA5BMMU2RHO6IBB";
+  secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12";
+  env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}";
+
+  storeUrl = "s3://my-cache?endpoint=http://server:9000&region=eu-west-1";
+
+in {
+  name = "nix-copy-closure";
+
+  nodes =
+    { server =
+        { config, lib, pkgs, ... }:
+        { virtualisation.writableStore = true;
+          virtualisation.additionalPaths = [ pkgA ];
+          environment.systemPackages = [ pkgs.minio-client ];
+          nix.extraOptions = "experimental-features = nix-command";
+          services.minio = {
+            enable = true;
+            region = "eu-west-1";
+            rootCredentialsFile = pkgs.writeText "minio-credentials-full" ''
+              MINIO_ROOT_USER=${accessKey}
+              MINIO_ROOT_PASSWORD=${secretKey}
+            '';
+          };
+          networking.firewall.allowedTCPPorts = [ 9000 ];
+        };
+
+      client =
+        { config, pkgs, ... }:
+        { virtualisation.writableStore = true;
+          nix.extraOptions = "experimental-features = nix-command";
+        };
+    };
+
+  testScript = { nodes }: ''
+    # fmt: off
+    start_all()
+
+    # Create a binary cache.
+    server.wait_for_unit("minio")
+
+    server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4")
+    server.succeed("mc mb minio/my-cache")
+
+    server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}")
+
+    # Test fetchurl on s3:// URLs while we're at it.
+    client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000&region=eu-west-1\"; }'")
+
+    # Copy a package from the binary cache.
+    client.fail("nix path-info ${pkgA}")
+
+    client.succeed("${env} nix store info --store '${storeUrl}' >&2")
+
+    client.succeed("${env} nix copy --no-check-sigs --from '${storeUrl}' ${pkgA}")
+
+    client.succeed("nix path-info ${pkgA}")
+  '';
+}