diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index bc38c72a8..b55b53899 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -497,13 +497,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this */ ref getRawAccessor( const Hash & rev, - bool smudgeLfs = false); + bool smudgeLfs = false, + bool applyFilters = false); ref getAccessor( const Hash & rev, bool exportIgnore, std::string displayPrefix, - bool smudgeLfs = false) override; + bool smudgeLfs = false, + bool applyFilters = false) override; ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override; @@ -668,20 +670,22 @@ struct GitSourceAccessor : SourceAccessor struct State { ref repo; - std::string gitRev; + git_oid oid; Object root; std::optional lfsFetch = std::nullopt; + bool applyFilters; }; Sync state_; - GitSourceAccessor(ref repo_, const Hash & rev, bool smudgeLfs) + GitSourceAccessor(ref repo_, const Hash & rev, bool smudgeLfs, bool applyFilters_) : state_{ State { .repo = repo_, - .gitRev = rev.gitRev(), + .oid = hashToOID(rev), .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()), .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt, + .applyFilters = applyFilters_, } } { @@ -709,28 +713,28 @@ struct GitSourceAccessor : SourceAccessor } } - // Apply git filters including CRLF conversion - git_buf filtered = GIT_BUF_INIT; - git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT; + if (!state->applyFilters) + return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); + else { + // Apply git filters including potential CRLF conversion + git_buf filtered = GIT_BUF_INIT; + git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT; - git_oid oid; - if (git_oid_fromstr(&oid, state->gitRev.c_str())) - throw Error("cannot convert '%s' to a Git OID", state->gitRev.c_str()); + opts.attr_commit_id = state->oid; + opts.flags = GIT_BLOB_FILTER_ATTRIBUTES_FROM_COMMIT; - opts.attr_commit_id = oid; - opts.flags = GIT_BLOB_FILTER_ATTRIBUTES_FROM_COMMIT; - - int error = git_blob_filter(&filtered, blob.get(), path.rel_c_str(), &opts); - if (error != 0) { - const git_error *e = git_error_last(); - std::string errorMsg = e ? e->message : "Unknown error"; + int error = git_blob_filter(&filtered, blob.get(), path.rel_c_str(), &opts); + if (error != 0) { + const git_error *e = git_error_last(); + std::string errorMsg = e ? e->message : "Unknown error"; + git_buf_dispose(&filtered); + throw Error("Failed to filter blob: " + errorMsg); + } + std::string result(filtered.ptr, filtered.size); git_buf_dispose(&filtered); - throw Error("Failed to filter blob: " + errorMsg); - } - std::string result(filtered.ptr, filtered.size); - git_buf_dispose(&filtered); - return result; + return result; + } } std::string readFile(const CanonPath & path) override @@ -1243,20 +1247,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink ref GitRepoImpl::getRawAccessor( const Hash & rev, - bool smudgeLfs) + bool smudgeLfs, + bool applyFilters) { auto self = ref(shared_from_this()); - return make_ref(self, rev, smudgeLfs); + return make_ref(self, rev, smudgeLfs, applyFilters); } ref GitRepoImpl::getAccessor( const Hash & rev, bool exportIgnore, std::string displayPrefix, - bool smudgeLfs) + bool smudgeLfs, + bool applyFilters) { auto self = ref(shared_from_this()); - ref rawGitAccessor = getRawAccessor(rev, smudgeLfs); + ref rawGitAccessor = getRawAccessor(rev, smudgeLfs, applyFilters); rawGitAccessor->setPathDisplay(std::move(displayPrefix)); if (exportIgnore) return make_ref(self, rawGitAccessor, rev); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index cf255c001..9d37c4ee8 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -182,7 +182,7 @@ struct GitInputScheme : InputScheme for (auto & [name, value] : url.query) { if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit") + else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit" || name == "applyFilters") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -220,6 +220,7 @@ struct GitInputScheme : InputScheme "keytype", "publicKey", "publicKeys", + "applyFilters", }; } @@ -275,6 +276,8 @@ struct GitInputScheme : InputScheme } else if (publicKeys.size() > 1) url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys)); + if (maybeGetBoolAttr(input.attrs, "applyFilters").value_or(false)) + url.query.insert_or_assign("applyFilters", "1"); return url; } @@ -425,6 +428,11 @@ struct GitInputScheme : InputScheme return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); } + bool getApplyFiltersAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "applyFilters").value_or(false); + } + RepoInfo getRepoInfo(const Input & input) const { auto checkHashAlgorithm = [&](const std::optional & hash) @@ -691,7 +699,8 @@ struct GitInputScheme : InputScheme bool exportIgnore = getExportIgnoreAttr(input); bool smudgeLfs = getLfsAttr(input); - auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs); + bool applyFilters = getApplyFiltersAttr(input); + auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs, applyFilters); /* If the repo has submodules, fetch them and return a mounted input accessor consisting of the accessor for the top-level @@ -710,6 +719,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("ref", submodule.branch); attrs.insert_or_assign("rev", submoduleRev.gitRev()); attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); + attrs.insert_or_assign("applyFilters", Explicit{ applyFilters }); attrs.insert_or_assign("submodules", Explicit{ true }); attrs.insert_or_assign("lfs", Explicit{ smudgeLfs }); attrs.insert_or_assign("allRefs", Explicit{ true }); @@ -854,7 +864,8 @@ struct GitInputScheme : InputScheme { auto makeFingerprint = [&](const Hash & rev) { - return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : ""); + // FIXME(gdennis): Update + return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : "") + (getApplyFiltersAttr(input) ? ";f" : ""); }; if (auto rev = input.getRev()) diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 2926deb4f..386cbe216 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -90,7 +90,8 @@ struct GitRepo const Hash & rev, bool exportIgnore, std::string displayPrefix, - bool smudgeLfs = false) = 0; + bool smudgeLfs = false, + bool applyFilters = false) = 0; virtual ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0; diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index ba08cef7a..49f1a51f5 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -318,12 +318,18 @@ echo 'test.txt eol=crlf' > "$repo/.gitattributes" git -C "$repo" add .gitattributes git -C "$repo" commit -m 'Add eol=crlf to gitattributes' narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash") +[[ "$narhash" = "sha256-BBhuj+vOnwCUnk5az22PwAnF32KE1aulWAVfCQlbW7U=" ]] + +narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; applyFilters = true; }).narHash") [[ "$narhash" = "sha256-k7u7RAaF+OvrbtT3KCCDQA8e9uOdflUo5zSgsosoLzA=" ]] + # Ensure that NAR hash doesn't depend on user configuration. rm -rf $TEST_HOME/.cache/nix export GIT_CONFIG_GLOBAL="$TEST_ROOT/gitconfig" git config --global core.autocrlf true -new_narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash") -[[ "$new_narhash" = "$narhash" ]] +narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash") +[[ "$narhash" = "sha256-BBhuj+vOnwCUnk5az22PwAnF32KE1aulWAVfCQlbW7U=" ]] +narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; applyFilters = true; }).narHash") +[[ "$narhash" = "sha256-k7u7RAaF+OvrbtT3KCCDQA8e9uOdflUo5zSgsosoLzA=" ]] unset GIT_CONFIG_GLOBAL