Merge pull request #117 from DeterminateSystems/test-uncacheable

Fix fetchToStore() caching with --impure, improve testing

src/libfetchers/fetch-to-store.cc

@@ -55,9 +55,13 @@ std::pair<StorePath, Hash> fetchToStore2(
             }
             debug("source path '%s' not in store", path);
         }
-    } else
+    } else {
+        static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
+        if (barf)
+            throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter);
         // FIXME: could still provide in-memory caching keyed on `SourcePath`.
-        debug("source path '%s' is uncacheable (%d, %d)", path, (bool) filter, (bool) fingerprint);
+        debug("source path '%s' is uncacheable", path);
+    }
 
     Activity act(*logger, lvlChatty, actUnknown,
         fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path));

src/libfetchers/git.cc

@@ -860,7 +860,7 @@ struct GitInputScheme : InputScheme
             return makeFingerprint(*rev);
         else {
             auto repoInfo = getRepoInfo(input);
-            if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) {
+            if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) {
                 /* Calculate a fingerprint that takes into account the
                    deleted and modified/added files. */
                 HashSink hashSink{HashAlgorithm::SHA512};
@@ -873,7 +873,7 @@ struct GitInputScheme : InputScheme
                     writeString("deleted:", hashSink);
                     writeString(file.abs(), hashSink);
                 }
-                return makeFingerprint(*repoInfo.workdirInfo.headRev)
+                return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev))
                     + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false);
             }
             return std::nullopt;

src/libutil/union-source-accessor.cc

@@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor
         }
         return std::nullopt;
     }
+
+    std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
+    {
+        if (fingerprint)
+            return {path, fingerprint};
+        for (auto & accessor : accessors) {
+            auto [subpath, fingerprint] = accessor->getFingerprint(path);
+            if (fingerprint)
+                return {subpath, fingerprint};
+        }
+        return {path, std::nullopt};
+    }
 };
 
 ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors)

tests/functional/flakes/common.sh

@@ -2,6 +2,8 @@
 
 source ../common.sh
 
+export _NIX_TEST_BARF_ON_UNCACHEABLE=1
+
 # shellcheck disable=SC2034 # this variable is used by tests that source this file
 registry=$TEST_ROOT/registry.json
 

tests/functional/flakes/flake-in-submodule.sh

@@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule
 # Check that dirtying a submodule makes the entire thing dirty.
 [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]]
 echo '"foo"' > "$rootRepo"/submodule/sub.nix
-[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]]
-[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]]
+[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]]
+[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]]
 
 # Test that `nix flake metadata` parses `submodule` correctly.
 cat > "$rootRepo"/flake.nix <<EOF
@@ -75,7 +75,7 @@ EOF
 git -C "$rootRepo" add flake.nix
 git -C "$rootRepo" commit -m "Add flake.nix"
 
-storePath=$(nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath)
+storePath=$(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake prefetch --json "$rootRepo?submodules=1" | jq -r .storePath)
 [[ -e "$storePath/submodule" ]]
 
 # Test the use of inputs.self.

tests/functional/flakes/flakes.sh

@@ -114,7 +114,6 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default"
 
 # Check that the fetcher cache works.
 if [[ $(nix config show lazy-trees) = false ]]; then
-    nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuietInverse "source path.*is uncacheable"
     nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path.*cache hit"
 fi
 

tests/functional/flakes/mercurial.sh

@@ -27,9 +27,9 @@ nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir"
 
 (! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision)
 
-nix eval "hg+file://$flake2Dir"#expr
+_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr
 
-nix eval "hg+file://$flake2Dir"#expr
+_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr
 
 (! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty)
 

tests/functional/flakes/non-flake-inputs.sh

@@ -72,7 +72,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file
 
 nix registry add --registry "$registry" flake3 "git+file://$flake3Dir"
 
-nix build -o "$TEST_ROOT/result" flake3#fnord
+_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord
 [[ $(cat "$TEST_ROOT/result") = FNORD ]]
 
 # Check whether flake input fetching is lazy: flake3#sth does not
@@ -82,11 +82,11 @@ clearStore
 mv "$flake2Dir" "$flake2Dir.tmp"
 mv "$nonFlakeDir" "$nonFlakeDir.tmp"
 nix build -o "$TEST_ROOT/result" flake3#sth
-(! nix build -o "$TEST_ROOT/result" flake3#xyzzy)
-(! nix build -o "$TEST_ROOT/result" flake3#fnord)
+(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy)
+(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord)
 mv "$flake2Dir.tmp" "$flake2Dir"
 mv "$nonFlakeDir.tmp" "$nonFlakeDir"
-nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
+_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord
 
 # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
 git -C "$flake3Dir" checkout -b removeXyzzy

tests/functional/flakes/relative-paths-lockfile.sh

@@ -4,6 +4,8 @@ source ./common.sh
 
 requireGit
 
+unset _NIX_TEST_BARF_ON_UNCACHEABLE
+
 # Test a "vendored" subflake dependency. This is a relative path flake
 # which doesn't reference the root flake and has its own lock file.
 #