Make it optional to apply git filters

src/libfetchers/git-utils.cc

@@ -497,13 +497,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
      */
     ref<GitSourceAccessor> getRawAccessor(
         const Hash & rev,
-        bool smudgeLfs = false);
+        bool smudgeLfs = false,
+        bool applyFilters = false);
 
     ref<SourceAccessor> getAccessor(
         const Hash & rev,
         bool exportIgnore,
         std::string displayPrefix,
-        bool smudgeLfs = false) override;
+        bool smudgeLfs = false,
+        bool applyFilters = false) override;
 
     ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
 
@@ -668,20 +670,22 @@ struct GitSourceAccessor : SourceAccessor
     struct State
     {
         ref<GitRepoImpl> repo;
-        std::string gitRev;
+        git_oid oid;
         Object root;
         std::optional<lfs::Fetch> lfsFetch = std::nullopt;
+        bool applyFilters;
     };
 
     Sync<State> state_;
 
-    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
+    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs, bool applyFilters_)
         : state_{
                 State {
                     .repo = repo_,
-                    .gitRev = rev.gitRev(),
+                    .oid = hashToOID(rev),
                     .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
                     .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+                    .applyFilters = applyFilters_,
                 }
             }
     {
@@ -709,15 +713,14 @@ struct GitSourceAccessor : SourceAccessor
             }
         }
 
-        // Apply git filters including CRLF conversion
+        if (!state->applyFilters)
+            return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+        else {
+            // Apply git filters including potential CRLF conversion
             git_buf filtered = GIT_BUF_INIT;
             git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT;
 
-        git_oid oid;
+            opts.attr_commit_id = state->oid;
-        if (git_oid_fromstr(&oid, state->gitRev.c_str()))
-            throw Error("cannot convert '%s' to a Git OID", state->gitRev.c_str());
-
-        opts.attr_commit_id = oid;
             opts.flags = GIT_BLOB_FILTER_ATTRIBUTES_FROM_COMMIT;
 
             int error = git_blob_filter(&filtered, blob.get(), path.rel_c_str(), &opts);
@@ -732,6 +735,7 @@ struct GitSourceAccessor : SourceAccessor
 
             return result;
         }
+    }
 
     std::string readFile(const CanonPath & path) override
     {
@@ -1243,20 +1247,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
 ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(
     const Hash & rev,
-    bool smudgeLfs)
+    bool smudgeLfs,
+    bool applyFilters)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
+    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs, applyFilters);
 }
 
 ref<SourceAccessor> GitRepoImpl::getAccessor(
     const Hash & rev,
     bool exportIgnore,
     std::string displayPrefix,
-    bool smudgeLfs)
+    bool smudgeLfs,
+    bool applyFilters)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
+    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs, applyFilters);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
     if (exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);

src/libfetchers/git.cc

@@ -182,7 +182,7 @@ struct GitInputScheme : InputScheme
         for (auto & [name, value] : url.query) {
             if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
                 attrs.emplace(name, value);
-            else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit")
+            else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit" || name == "applyFilters")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -220,6 +220,7 @@ struct GitInputScheme : InputScheme
             "keytype",
             "publicKey",
             "publicKeys",
+            "applyFilters",
         };
     }
 
@@ -275,6 +276,8 @@ struct GitInputScheme : InputScheme
         }
         else if (publicKeys.size() > 1)
             url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys));
+        if (maybeGetBoolAttr(input.attrs, "applyFilters").value_or(false))
+            url.query.insert_or_assign("applyFilters", "1");
         return url;
     }
 
@@ -425,6 +428,11 @@ struct GitInputScheme : InputScheme
         return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
     }
 
+    bool getApplyFiltersAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "applyFilters").value_or(false);
+    }
+
     RepoInfo getRepoInfo(const Input & input) const
     {
         auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
@@ -691,7 +699,8 @@ struct GitInputScheme : InputScheme
 
         bool exportIgnore = getExportIgnoreAttr(input);
         bool smudgeLfs = getLfsAttr(input);
-        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
+        bool applyFilters = getApplyFiltersAttr(input);
+        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs, applyFilters);
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -710,6 +719,7 @@ struct GitInputScheme : InputScheme
                     attrs.insert_or_assign("ref", submodule.branch);
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
+                attrs.insert_or_assign("applyFilters", Explicit<bool>{ applyFilters });
                 attrs.insert_or_assign("submodules", Explicit<bool>{ true });
                 attrs.insert_or_assign("lfs", Explicit<bool>{ smudgeLfs });
                 attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
@@ -854,7 +864,8 @@ struct GitInputScheme : InputScheme
     {
         auto makeFingerprint = [&](const Hash & rev)
         {
-            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : "");
+            // FIXME(gdennis): Update
+            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : "") + (getApplyFiltersAttr(input) ? ";f" : "");
         };
 
         if (auto rev = input.getRev())

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -90,7 +90,8 @@ struct GitRepo
         const Hash & rev,
         bool exportIgnore,
         std::string displayPrefix,
-        bool smudgeLfs = false) = 0;
+        bool smudgeLfs = false,
+        bool applyFilters = false) = 0;
 
     virtual ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
 

tests/functional/fetchGit.sh

@@ -318,12 +318,18 @@ echo 'test.txt eol=crlf' > "$repo/.gitattributes"
 git -C "$repo" add .gitattributes
 git -C "$repo" commit -m 'Add eol=crlf to gitattributes'
 narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash")
+[[ "$narhash" = "sha256-BBhuj+vOnwCUnk5az22PwAnF32KE1aulWAVfCQlbW7U=" ]]
+
+narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; applyFilters = true; }).narHash")
 [[ "$narhash" = "sha256-k7u7RAaF+OvrbtT3KCCDQA8e9uOdflUo5zSgsosoLzA=" ]]
 
+
 # Ensure that NAR hash doesn't depend on user configuration.
 rm -rf $TEST_HOME/.cache/nix
 export GIT_CONFIG_GLOBAL="$TEST_ROOT/gitconfig"
 git config --global core.autocrlf true
-new_narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash")
+narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash")
-[[ "$new_narhash" = "$narhash" ]]
+[[ "$narhash" = "sha256-BBhuj+vOnwCUnk5az22PwAnF32KE1aulWAVfCQlbW7U=" ]]
+narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; applyFilters = true; }).narHash")
+[[ "$narhash" = "sha256-k7u7RAaF+OvrbtT3KCCDQA8e9uOdflUo5zSgsosoLzA=" ]]
 unset GIT_CONFIG_GLOBAL