Merge pull request #11992 from DeterminateSystems/dirty-git-fingerprint

Git fetcher: Calculate a fingerprint for dirty workdirs

src/libfetchers/fetchers.cc

@@ -113,7 +113,15 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
 
 std::optional<std::string> Input::getFingerprint(ref<Store> store) const
 {
-    return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
+    if (!scheme) return std::nullopt;
+
+    if (cachedFingerprint) return *cachedFingerprint;
+
+    auto fingerprint = scheme->getFingerprint(store, *this);
+
+    cachedFingerprint = fingerprint;
+
+    return fingerprint;
 }
 
 ParsedURL Input::toURL() const
@@ -307,7 +315,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
             auto accessor = makeStorePathAccessor(store, storePath);
 
-            accessor->fingerprint = scheme->getFingerprint(store, *this);
+            accessor->fingerprint = getFingerprint(store);
 
             return {accessor, *this};
         } catch (Error & e) {
@@ -318,7 +326,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
     auto [accessor, result] = scheme->getAccessor(store, *this);
 
     assert(!accessor->fingerprint);
-    accessor->fingerprint = scheme->getFingerprint(store, result);
+    accessor->fingerprint = result.getFingerprint(store);
 
     return {accessor, std::move(result)};
 }

src/libfetchers/fetchers.hh

@@ -46,6 +46,11 @@ struct Input
      */
     std::optional<Path> parent;
 
+    /**
+     * Cached result of getFingerprint().
+     */
+    mutable std::optional<std::optional<std::string>> cachedFingerprint;
+
 public:
     /**
      * Create an `Input` from a URL.

src/libfetchers/git-utils.cc

@@ -5,6 +5,7 @@
 #include "signals.hh"
 #include "users.hh"
 #include "fs-sink.hh"
+#include "sync.hh"
 
 #include <git2/attr.h>
 #include <git2/blob.h>
@@ -437,7 +438,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         {
             if (!(statusFlags & GIT_STATUS_INDEX_DELETED) &&
                 !(statusFlags & GIT_STATUS_WT_DELETED))
+            {
                 info.files.insert(CanonPath(path));
+                if (statusFlags != GIT_STATUS_CURRENT)
+                    info.dirtyFiles.insert(CanonPath(path));
+            } else
+                info.deletedFiles.insert(CanonPath(path));
             if (statusFlags != GIT_STATUS_CURRENT)
                 info.isDirty = true;
             return 0;
@@ -1262,4 +1268,17 @@ ref<GitRepo> getTarballCache()
     return GitRepo::openRepo(repoDir, true, true);
 }
 
+GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)
+{
+    static Sync<std::map<std::filesystem::path, WorkdirInfo>> _cache;
+    {
+        auto cache(_cache.lock());
+        auto i = cache->find(path);
+        if (i != cache->end()) return i->second;
+    }
+    auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo();
+    _cache.lock()->emplace(path, workdirInfo);
+    return workdirInfo;
+}
+
 }

src/libfetchers/git-utils.hh

@@ -59,12 +59,20 @@ struct GitRepo
            modified or added, but excluding deleted files. */
         std::set<CanonPath> files;
 
+        /* All modified or added files. */
+        std::set<CanonPath> dirtyFiles;
+
+        /* The deleted files. */
+        std::set<CanonPath> deletedFiles;
+
         /* The submodules listed in .gitmodules of this workdir. */
         std::vector<Submodule> submodules;
     };
 
     virtual WorkdirInfo getWorkdirInfo() = 0;
 
+    static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path);
+
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;
 

src/libfetchers/git.cc

@@ -15,6 +15,7 @@
 #include "finally.hh"
 #include "fetch-settings.hh"
 #include "json-utils.hh"
+#include "archive.hh"
 
 #include <regex>
 #include <string.h>
@@ -430,7 +431,7 @@ struct GitInputScheme : InputScheme
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.
         if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
-            repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo();
+            repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(repoInfo.url);
 
         return repoInfo;
     }
@@ -793,11 +794,34 @@ struct GitInputScheme : InputScheme
 
     std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
+        auto makeFingerprint = [&](const Hash & rev)
+        {
+            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
+        };
+
         if (auto rev = input.getRev())
-            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
-        else
+            return makeFingerprint(*rev);
+        else {
+            auto repoInfo = getRepoInfo(input);
+            if (repoInfo.isLocal && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) {
+                /* Calculate a fingerprint that takes into account the
+                   deleted and modified/added files. */
+                HashSink hashSink{HashAlgorithm::SHA512};
+                for (auto & file : repoInfo.workdirInfo.dirtyFiles) {
+                    writeString("modified:", hashSink);
+                    writeString(file.abs(), hashSink);
+                    dumpPath(repoInfo.url + "/" + file.abs(), hashSink);
+                }
+                for (auto & file : repoInfo.workdirInfo.deletedFiles) {
+                    writeString("deleted:", hashSink);
+                    writeString(file.abs(), hashSink);
+                }
+                return makeFingerprint(*repoInfo.workdirInfo.headRev)
+                    + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false);
+            }
             return std::nullopt;
         }
+    }
 
     bool isLocked(const Input & input) const override
     {

tests/functional/flakes/flakes.sh

@@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision)
 echo foo > "$flake1Dir/foo"
 git -C "$flake1Dir" add $flake1Dir/foo
 [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]]
+[[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]]
 
 echo -n '# foo' >> "$flake1Dir/flake.nix"
 flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD)