From 1b2cb1d75c919122ff5b6a13ff47febf86020b6f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 2 Dec 2025 06:09:03 +0300 Subject: [PATCH] libfetchers/git-utils: Only create pack and mempack backends for the tarball cache Now the unnecessary utimensat syscalls from the previous commit are completely gone: % time seconds usecs/call calls errors syscall ------ ----------- ----------- --------- --------- ------------------ 33.39 0.646359 3 162898 getdents64 29.34 0.567866 3 163523 81934 openat 14.81 0.286739 3 81835 203 newfstatat 10.98 0.212550 2 81593 close 10.56 0.204458 2 81544 fstat 0.15 0.002814 3 870 mmap The rather crazy amount of getdents64 is still there though. --- src/libfetchers/git-utils.cc | 54 +++++++++++++++---- .../include/nix/fetchers/git-utils.hh | 3 +- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 90c2e1a4f..aa9efb883 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -245,9 +247,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this * In-memory object store for efficient batched writing to packfiles. * Owned by `repo`. */ - git_odb_backend * mempack_backend; + git_odb_backend * mempackBackend = nullptr; - GitRepoImpl(std::filesystem::path _path, bool create, bool bare) + /** + * On-disk packfile object store. + * Owned by `repo`. + */ + git_odb_backend * packBackend = nullptr; + + GitRepoImpl(std::filesystem::path _path, bool create, bool bare, bool packfilesOnly = false) : path(std::move(_path)) , bare(bare) { @@ -258,15 +266,39 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this throw Error("opening Git repository %s: %s", path, git_error_last()->message); ObjectDb odb; - if (git_repository_odb(Setter(odb), repo.get())) - throw Error("getting Git object database: %s", git_error_last()->message); + if (packfilesOnly) { + /* Create a fresh object database because by default the repo also + loose object backends. We are not using any of those for the + tarball cache, but libgit2 still does a bunch of unnecessary + syscalls that always fail with ENOENT. NOTE: We are only creating + a libgit2 object here and not modifying the repo. Think of this as + enabling the specific backend. + */ + + if (git_odb_new(Setter(odb))) + throw Error("creating Git object database: %s", git_error_last()->message); + + if (git_odb_backend_pack(&packBackend, (path / "objects").string().c_str())) + throw Error("creating pack backend: %s", git_error_last()->message); + + if (git_odb_add_backend(odb.get(), packBackend, 1)) + throw Error("adding pack backend to Git object database: %s", git_error_last()->message); + } else { + if (git_repository_odb(Setter(odb), repo.get())) + throw Error("getting Git object database: %s", git_error_last()->message); + } // mempack_backend will be owned by the repository, so we are not expected to free it ourselves. - if (git_mempack_new(&mempack_backend)) + if (git_mempack_new(&mempackBackend)) throw Error("creating mempack backend: %s", git_error_last()->message); - if (git_odb_add_backend(odb.get(), mempack_backend, 999)) + if (git_odb_add_backend(odb.get(), mempackBackend, 999)) throw Error("adding mempack backend to Git object database: %s", git_error_last()->message); + + if (packfilesOnly) { + if (git_repository_set_odb(repo.get(), odb.get())) + throw Error("setting Git object database: %s", git_error_last()->message); + } } operator git_repository *() @@ -287,7 +319,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this git_packbuilder_set_threads(packBuilder.get(), 0 /* autodetect */); packBuilderContext.handleException( - "preparing packfile", git_mempack_write_thin_pack(mempack_backend, packBuilder.get())); + "preparing packfile", git_mempack_write_thin_pack(mempackBackend, packBuilder.get())); checkInterrupt(); packBuilderContext.handleException("writing packfile", git_packbuilder_write_buf(&buf, packBuilder.get())); checkInterrupt(); @@ -320,7 +352,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_indexer_commit(indexer.get(), &stats)) throw Error("committing git packfile index: %s", git_error_last()->message); - if (git_mempack_reset(mempack_backend)) + if (git_mempack_reset(mempackBackend)) throw Error("resetting git mempack backend: %s", git_error_last()->message); checkInterrupt(); @@ -680,9 +712,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this } }; -ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) +ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare, bool packfilesOnly) { - return make_ref(path, create, bare); + return make_ref(path, create, bare, packfilesOnly); } /** @@ -1361,7 +1393,7 @@ namespace fetchers { ref Settings::getTarballCache() const { static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; - return GitRepo::openRepo(repoDir, true, true); + return GitRepo::openRepo(repoDir, /*create=*/true, /*bare=*/true, /*packfilesOnly=*/true); } } // namespace fetchers diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 8ed48a7b5..5c79f256e 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -32,7 +32,8 @@ struct GitRepo { virtual ~GitRepo() {} - static ref openRepo(const std::filesystem::path & path, bool create = false, bool bare = false); + static ref + openRepo(const std::filesystem::path & path, bool create = false, bool bare = false, bool packfilesOnly = false); virtual uint64_t getRevCount(const Hash & rev) = 0;