nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-28 13:11:00 +01:00
Code Activity

Make it optional to apply git filters

Browse source
This commit is contained in:
Graham Dennis 2025-09-15 10:33:22 +10:00
parent d62f504799
commit b78a8d9add
4 changed files with 57 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

60
src/libfetchers/git-utils.cc
View file

@ -497,13 +497,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
*/
ref<GitSourceAccessor> getRawAccessor(
const Hash & rev,
bool smudgeLfs = false);
bool smudgeLfs = false,
bool applyFilters = false);
ref<SourceAccessor> getAccessor(
const Hash & rev,
bool exportIgnore,
std::string displayPrefix,
bool smudgeLfs = false) override;
bool smudgeLfs = false,
bool applyFilters = false) override;
ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
@ -668,20 +670,22 @@ struct GitSourceAccessor : SourceAccessor
struct State
{
ref<GitRepoImpl> repo;
std::string gitRev;
git_oid oid;
Object root;
std::optional<lfs::Fetch> lfsFetch = std::nullopt;
bool applyFilters;
};
Sync<State> state_;
GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs, bool applyFilters_)
: state_{
State {
.repo = repo_,
.gitRev = rev.gitRev(),
.oid = hashToOID(rev),
.root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
.lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
.applyFilters = applyFilters_,
}
}
{
@ -709,28 +713,28 @@ struct GitSourceAccessor : SourceAccessor
}
}
// Apply git filters including CRLF conversion
git_buf filtered = GIT_BUF_INIT;
git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT;
if (!state->applyFilters)
return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
else {
// Apply git filters including potential CRLF conversion
git_buf filtered = GIT_BUF_INIT;
git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT;
git_oid oid;
if (git_oid_fromstr(&oid, state->gitRev.c_str()))
throw Error("cannot convert '%s' to a Git OID", state->gitRev.c_str());
opts.attr_commit_id = state->oid;
opts.flags = GIT_BLOB_FILTER_ATTRIBUTES_FROM_COMMIT;
opts.attr_commit_id = oid;
opts.flags = GIT_BLOB_FILTER_ATTRIBUTES_FROM_COMMIT;
int error = git_blob_filter(&filtered, blob.get(), path.rel_c_str(), &opts);
if (error != 0) {
const git_error *e = git_error_last();
std::string errorMsg = e ? e->message : "Unknown error";
int error = git_blob_filter(&filtered, blob.get(), path.rel_c_str(), &opts);
if (error != 0) {
const git_error *e = git_error_last();
std::string errorMsg = e ? e->message : "Unknown error";
git_buf_dispose(&filtered);
throw Error("Failed to filter blob: " + errorMsg);
}
std::string result(filtered.ptr, filtered.size);
git_buf_dispose(&filtered);
throw Error("Failed to filter blob: " + errorMsg);
}
std::string result(filtered.ptr, filtered.size);
git_buf_dispose(&filtered);
return result;
return result;
}
}
std::string readFile(const CanonPath & path) override
@ -1243,20 +1247,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(
const Hash & rev,
bool smudgeLfs)
bool smudgeLfs,
bool applyFilters)
{
auto self = ref<GitRepoImpl>(shared_from_this());
return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
return make_ref<GitSourceAccessor>(self, rev, smudgeLfs, applyFilters);
}
ref<SourceAccessor> GitRepoImpl::getAccessor(
const Hash & rev,
bool exportIgnore,
std::string displayPrefix,
bool smudgeLfs)
bool smudgeLfs,
bool applyFilters)
{
auto self = ref<GitRepoImpl>(shared_from_this());
ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs, applyFilters);
rawGitAccessor->setPathDisplay(std::move(displayPrefix));
if (exportIgnore)
return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);

17
src/libfetchers/git.cc
View file

@ -182,7 +182,7 @@ struct GitInputScheme : InputScheme
for (auto & [name, value] : url.query) {
if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
attrs.emplace(name, value);
else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit")
else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit" || name == "applyFilters")
attrs.emplace(name, Explicit<bool> { value == "1" });
else
url2.query.emplace(name, value);
@ -220,6 +220,7 @@ struct GitInputScheme : InputScheme
"keytype",
"publicKey",
"publicKeys",
"applyFilters",
};
}
@ -275,6 +276,8 @@ struct GitInputScheme : InputScheme
}
else if (publicKeys.size() > 1)
url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys));
if (maybeGetBoolAttr(input.attrs, "applyFilters").value_or(false))
url.query.insert_or_assign("applyFilters", "1");
return url;
}
@ -425,6 +428,11 @@ struct GitInputScheme : InputScheme
return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
}
bool getApplyFiltersAttr(const Input & input) const
{
return maybeGetBoolAttr(input.attrs, "applyFilters").value_or(false);
}
RepoInfo getRepoInfo(const Input & input) const
{
auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
@ -691,7 +699,8 @@ struct GitInputScheme : InputScheme
bool exportIgnore = getExportIgnoreAttr(input);
bool smudgeLfs = getLfsAttr(input);
auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
bool applyFilters = getApplyFiltersAttr(input);
auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs, applyFilters);
/* If the repo has submodules, fetch them and return a mounted
input accessor consisting of the accessor for the top-level
@ -710,6 +719,7 @@ struct GitInputScheme : InputScheme
attrs.insert_or_assign("ref", submodule.branch);
attrs.insert_or_assign("rev", submoduleRev.gitRev());
attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
attrs.insert_or_assign("applyFilters", Explicit<bool>{ applyFilters });
attrs.insert_or_assign("submodules", Explicit<bool>{ true });
attrs.insert_or_assign("lfs", Explicit<bool>{ smudgeLfs });
attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
@ -854,7 +864,8 @@ struct GitInputScheme : InputScheme
{
auto makeFingerprint = [&](const Hash & rev)
{
return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : "");
// FIXME(gdennis): Update
return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : "") + (getApplyFiltersAttr(input) ? ";f" : "");
};
if (auto rev = input.getRev())

3
src/libfetchers/include/nix/fetchers/git-utils.hh
View file

@ -90,7 +90,8 @@ struct GitRepo
const Hash & rev,
bool exportIgnore,
std::string displayPrefix,
bool smudgeLfs = false) = 0;
bool smudgeLfs = false,
bool applyFilters = false) = 0;
virtual ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;

10
tests/functional/fetchGit.sh
View file

@ -318,12 +318,18 @@ echo 'test.txt eol=crlf' > "$repo/.gitattributes"
git -C "$repo" add .gitattributes
git -C "$repo" commit -m 'Add eol=crlf to gitattributes'
narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash")
[[ "$narhash" = "sha256-BBhuj+vOnwCUnk5az22PwAnF32KE1aulWAVfCQlbW7U=" ]]
narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; applyFilters = true; }).narHash")
[[ "$narhash" = "sha256-k7u7RAaF+OvrbtT3KCCDQA8e9uOdflUo5zSgsosoLzA=" ]]
# Ensure that NAR hash doesn't depend on user configuration.
rm -rf $TEST_HOME/.cache/nix
export GIT_CONFIG_GLOBAL="$TEST_ROOT/gitconfig"
git config --global core.autocrlf true
new_narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash")
[[ "$new_narhash" = "$narhash" ]]
narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; }).narHash")
[[ "$narhash" = "sha256-BBhuj+vOnwCUnk5az22PwAnF32KE1aulWAVfCQlbW7U=" ]]
narhash=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = \"$repo\"; ref = \"master\"; applyFilters = true; }).narHash")
[[ "$narhash" = "sha256-k7u7RAaF+OvrbtT3KCCDQA8e9uOdflUo5zSgsosoLzA=" ]]
unset GIT_CONFIG_GLOBAL