From 4f8c50fb77facc9cc1e574130fdca3ea502ab518 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Tue, 5 Aug 2025 00:55:32 +0200 Subject: [PATCH 1/3] libexpr: replace std::unordered_* types by faster boost hash maps --- src/libexpr/eval.cc | 24 +++++++-------- src/libexpr/include/nix/expr/eval.hh | 24 ++++++++------- src/libexpr/include/nix/expr/parser-state.hh | 2 +- src/libexpr/include/nix/expr/value.hh | 3 +- src/libexpr/parser.y | 2 +- src/libexpr/primops.cc | 32 ++++++++------------ 6 files changed, 42 insertions(+), 45 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index df4e52e5d..69d7ba380 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1090,7 +1090,9 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) void EvalState::resetFileCache() { fileEvalCache.clear(); + fileEvalCache.rehash(0); fileParseCache.clear(); + fileParseCache.rehash(0); inputCache->clear(); } @@ -2375,10 +2377,9 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - auto dstPathCached = get(*srcToStore.lock(), path); - - auto dstPath = dstPathCached ? *dstPathCached : [&]() { - auto dstPath = fetchToStore( + std::optional dstPath; + if (!srcToStore.cvisit(path, [&dstPath](const auto & kv) { dstPath.emplace(kv.second); })) { + dstPath.emplace(fetchToStore( fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), @@ -2386,15 +2387,14 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat path.baseName(), ContentAddressMethod::Raw::NixArchive, nullptr, - repair); - allowPath(dstPath); - srcToStore.lock()->try_emplace(path, dstPath); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); - return dstPath; - }(); + repair)); + allowPath(*dstPath); + srcToStore.try_emplace(path, *dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(*dstPath)); + } - context.insert(NixStringContextElem::Opaque{.path = dstPath}); - return dstPath; + context.insert(NixStringContextElem::Opaque{.path = *dstPath}); + return *dstPath; } SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 5015a009b..75ed12664 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -20,6 +20,8 @@ // For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS` #include "nix/expr/config.hh" +#include +#include #include #include #include @@ -162,7 +164,7 @@ typedef std:: map, traceable_allocator>> ValMap; -typedef std::unordered_map DocCommentMap; +typedef boost::unordered_flat_map> DocCommentMap; struct Env { @@ -395,7 +397,7 @@ public: bool inDebugger = false; int trylevel; std::list debugTraces; - std::map> exprEnvs; + boost::unordered_flat_map> exprEnvs; const std::shared_ptr getStaticEnv(const Expr & expr) const { @@ -438,12 +440,12 @@ private: /* Cache for calls to addToStore(); maps source paths to the store paths. */ - Sync> srcToStore; + boost::concurrent_flat_map> srcToStore; /** * A cache from path names to parse trees. */ - typedef std::unordered_map< + typedef boost::unordered_flat_map< SourcePath, Expr *, std::hash, @@ -455,7 +457,7 @@ private: /** * A cache from path names to values. */ - typedef std::unordered_map< + typedef boost::unordered_flat_map< SourcePath, Value, std::hash, @@ -468,11 +470,11 @@ private: * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. * Grouped by file. */ - std::unordered_map positionToDocComment; + boost::unordered_flat_map> positionToDocComment; LookupPath lookupPath; - std::map> lookupPathResolved; + boost::unordered_flat_map> lookupPathResolved; /** * Cache used by prim_match(). @@ -746,7 +748,7 @@ public: /** * Internal primops not exposed to the user. */ - std::unordered_map< + boost::unordered_flat_map< std::string, Value *, std::hash, @@ -1017,10 +1019,10 @@ private: bool countCalls; - typedef std::map PrimOpCalls; + typedef boost::unordered_flat_map PrimOpCalls; PrimOpCalls primOpCalls; - typedef std::map FunctionCalls; + typedef boost::unordered_flat_map FunctionCalls; FunctionCalls functionCalls; /** Evaluation/call profiler. */ @@ -1028,7 +1030,7 @@ private: void incrFunctionCall(ExprLambda * fun); - typedef std::map AttrSelects; + typedef boost::unordered_flat_map> AttrSelects; AttrSelects attrSelects; friend struct ExprOpUpdate; diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index 836cc9861..e689678de 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -71,7 +71,7 @@ struct LexerState /** * @brief Maps some positions to a DocComment, where the comment is relevant to the location. */ - std::unordered_map & positionToDocComment; + DocCommentMap & positionToDocComment; PosTable & positions; PosTable::Origin origin; diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 82db1a775..d3aeac157 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -12,6 +12,7 @@ #include "nix/expr/print-options.hh" #include "nix/util/checked-arithmetic.hh" +#include #include namespace nix { @@ -1166,7 +1167,7 @@ void Value::mkBlackhole() } typedef std::vector> ValueVector; -typedef std::unordered_map< +typedef boost::unordered_flat_map< Symbol, Value *, std::hash, diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 35fe929d9..89da001ef 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -57,7 +57,7 @@ namespace nix { -typedef std::unordered_map DocCommentMap; +typedef boost::unordered_flat_map> DocCommentMap; Expr * parseExprFromBuf( char * text, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9ba417c32..c107c6bc2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -18,6 +18,8 @@ #include "nix/util/sort.hh" #include +#include +#include #include #include @@ -1750,7 +1752,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName read them later. */ { auto h = hashDerivationModulo(*state.store, drv, false); - drvHashes.lock()->insert_or_assign(drvPath, h); + drvHashes.insert_or_assign(drvPath, std::move(h)); } auto result = state.buildBindings(1 + drv.outputs.size()); @@ -4027,7 +4029,7 @@ static void prim_groupBy(EvalState & state, const PosIdx pos, Value ** args, Val auto name = state.forceStringNoCtx( res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy"); auto sym = state.symbols.create(name); - auto vector = attrs.try_emplace(sym, ValueVector()).first; + auto vector = attrs.try_emplace(sym, {}).first; vector->second.push_back(vElem); } @@ -4562,27 +4564,19 @@ static RegisterPrimOp primop_convertHash({ struct RegexCache { - struct State - { - std::unordered_map> cache; - }; - - Sync state_; + boost::concurrent_flat_map> cache; std::regex get(std::string_view re) { - auto state(state_.lock()); - auto it = state->cache.find(re); - if (it != state->cache.end()) - return it->second; + std::regex regex; /* No std::regex constructor overload from std::string_view, but can be constructed from a pointer + size or an iterator range. */ - return state->cache - .emplace( - std::piecewise_construct, - std::forward_as_tuple(re), - std::forward_as_tuple(/*s=*/re.data(), /*count=*/re.size(), std::regex::extended)) - .first->second; + cache.try_emplace_and_cvisit(re, + /*s=*/re.data(), /*count=*/re.size(), std::regex::extended, + [®ex](const auto & kv) { regex = kv.second; }, + [®ex](const auto & kv) { regex = kv.second; } + ); + return regex; } }; @@ -4826,7 +4820,7 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value ** ar from.emplace_back(state.forceString( *elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::unordered_map cache; + boost::unordered_flat_map cache; auto to = args[1]->listView(); NixStringContext context; From 9f2b6a1b94141dc50deb6e1976beb6cf0872b33c Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 6 Sep 2025 14:21:48 +0200 Subject: [PATCH 2/3] replace more std::unordered_* types by faster boost hash maps --- src/libexpr-c/nix_api_expr.cc | 28 +++++++------------ src/libexpr/primops.cc | 10 ++++--- src/libexpr/print.cc | 5 ++-- src/libfetchers/filtering-source-accessor.cc | 8 ++++-- src/libfetchers/git-utils.cc | 11 ++++---- .../nix/fetchers/filtering-source-accessor.hh | 4 +-- src/libflake/lockfile.cc | 5 ++-- src/libstore/derivations.cc | 14 ++++------ src/libstore/gc.cc | 17 +++++------ src/libstore/include/nix/store/derivations.hh | 16 +++++++++-- src/libstore/include/nix/store/gc-store.hh | 6 ++-- src/libstore/include/nix/store/local-store.hh | 4 +-- src/libstore/misc.cc | 6 ++-- src/libutil/include/nix/util/canon-path.hh | 4 ++- src/libutil/linux/cgroup.cc | 4 +-- src/libutil/posix-source-accessor.cc | 2 +- src/nix/env.cc | 2 +- 17 files changed, 75 insertions(+), 71 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index a028202ae..46e08b5f7 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -16,7 +16,7 @@ #include "nix_api_util_internal.h" #if NIX_USE_BOEHMGC -# include +# include #endif /** @@ -207,28 +207,20 @@ void nix_state_free(EvalState * state) } #if NIX_USE_BOEHMGC -std::unordered_map< +boost::concurrent_flat_map< const void *, unsigned int, std::hash, std::equal_to, traceable_allocator>> - nix_refcounts; - -std::mutex nix_refcount_lock; + nix_refcounts{}; nix_err nix_gc_incref(nix_c_context * context, const void * p) { if (context) context->last_err_code = NIX_OK; try { - std::scoped_lock lock(nix_refcount_lock); - auto f = nix_refcounts.find(p); - if (f != nix_refcounts.end()) { - f->second++; - } else { - nix_refcounts[p] = 1; - } + nix_refcounts.insert_or_visit({p, 1}, [](auto & kv) { kv.second++; }); } NIXC_CATCH_ERRS } @@ -239,12 +231,12 @@ nix_err nix_gc_decref(nix_c_context * context, const void * p) if (context) context->last_err_code = NIX_OK; try { - std::scoped_lock lock(nix_refcount_lock); - auto f = nix_refcounts.find(p); - if (f != nix_refcounts.end()) { - if (--f->second == 0) - nix_refcounts.erase(f); - } else + bool fail = true; + nix_refcounts.erase_if(p, [&](auto & kv) { + fail = false; + return !--kv.second; + }); + if (fail) throw std::runtime_error("nix_gc_decref: object was not referenced"); } NIXC_CATCH_ERRS diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c107c6bc2..f2520bcda 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4571,11 +4571,13 @@ struct RegexCache std::regex regex; /* No std::regex constructor overload from std::string_view, but can be constructed from a pointer + size or an iterator range. */ - cache.try_emplace_and_cvisit(re, - /*s=*/re.data(), /*count=*/re.size(), std::regex::extended, + cache.try_emplace_and_cvisit( + re, + /*s=*/re.data(), + /*count=*/re.size(), + std::regex::extended, [®ex](const auto & kv) { regex = kv.second; }, - [®ex](const auto & kv) { regex = kv.second; } - ); + [®ex](const auto & kv) { regex = kv.second; }); return regex; } }; diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 5338e365e..071addc1a 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -1,5 +1,4 @@ #include -#include #include #include "nix/expr/print.hh" @@ -10,6 +9,8 @@ #include "nix/util/english.hh" #include "nix/expr/eval.hh" +#include + namespace nix { void printElided( @@ -81,7 +82,7 @@ std::ostream & printLiteralBool(std::ostream & str, bool boolean) // For example `or' doesn't need to be quoted. bool isReservedKeyword(const std::string_view str) { - static const std::unordered_set reservedKeywords = { + static const boost::unordered_flat_set reservedKeywords = { "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit"}; return reservedKeywords.contains(str); } diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 17f224ad2..d0991ae23 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -1,5 +1,7 @@ #include "nix/fetchers/filtering-source-accessor.hh" +#include + namespace nix { std::optional FilteringSourceAccessor::getPhysicalPath(const CanonPath & path) @@ -57,12 +59,12 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path) struct AllowListSourceAccessorImpl : AllowListSourceAccessor { std::set allowedPrefixes; - std::unordered_set allowedPaths; + boost::unordered_flat_set> allowedPaths; AllowListSourceAccessorImpl( ref next, std::set && allowedPrefixes, - std::unordered_set && allowedPaths, + boost::unordered_flat_set> && allowedPaths, MakeNotAllowedError && makeNotAllowedError) : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError)) , allowedPrefixes(std::move(allowedPrefixes)) @@ -84,7 +86,7 @@ struct AllowListSourceAccessorImpl : AllowListSourceAccessor ref AllowListSourceAccessor::create( ref next, std::set && allowedPrefixes, - std::unordered_set && allowedPaths, + boost::unordered_flat_set> && allowedPaths, MakeNotAllowedError && makeNotAllowedError) { return make_ref( diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 1861838ed..4ed94a4ed 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -30,8 +30,9 @@ #include #include +#include +#include #include -#include #include #include #include @@ -315,7 +316,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this uint64_t getRevCount(const Hash & rev) override { - std::unordered_set done; + boost::unordered_flat_set> done; std::queue todo; todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); @@ -569,7 +570,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this void verifyCommit(const Hash & rev, const std::vector & publicKeys) override { // Map of SSH key types to their internal OpenSSH representations - static const std::unordered_map keyTypeMap = { + static const boost::unordered_flat_map keyTypeMap = { {"ssh-dsa", "ssh-dsa"}, {"ssh-ecdsa", "ssh-ecdsa"}, {"ssh-ecdsa-sk", "sk-ecdsa-sha2-nistp256@openssh.com"}, @@ -816,7 +817,7 @@ struct GitSourceAccessor : SourceAccessor return toHash(*git_tree_entry_id(entry)); } - std::unordered_map lookupCache; + boost::unordered_flat_map> lookupCache; /* Recursively look up 'path' relative to the root. */ git_tree_entry * lookup(State & state, const CanonPath & path) @@ -1253,7 +1254,7 @@ GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllow makeFSSourceAccessor(path), std::set{wd.files}, // Always allow access to the root, but not its children. - std::unordered_set{CanonPath::root}, + boost::unordered_flat_set>{CanonPath::root}, std::move(makeNotAllowedError)) .cast(); if (exportIgnore) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 70e837ff4..1d4028be5 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -2,7 +2,7 @@ #include "nix/util/source-path.hh" -#include +#include namespace nix { @@ -72,7 +72,7 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor static ref create( ref next, std::set && allowedPrefixes, - std::unordered_set && allowedPaths, + boost::unordered_flat_set> && allowedPaths, MakeNotAllowedError && makeNotAllowedError); using FilteringSourceAccessor::FilteringSourceAccessor; diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 94e7f11f1..f381a57e6 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -1,5 +1,3 @@ -#include - #include "nix/fetchers/fetch-settings.hh" #include "nix/flake/settings.hh" #include "nix/flake/lockfile.hh" @@ -9,6 +7,7 @@ #include #include +#include #include #include @@ -162,7 +161,7 @@ std::pair LockFile::toJSON() const { nlohmann::json nodes; KeyMap nodeKeys; - std::unordered_set keys; + boost::unordered_flat_set keys; std::function node)> dumpNode; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a1831efc6..84889ceac 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -11,6 +11,7 @@ #include "nix/util/json-utils.hh" #include +#include #include namespace nix { @@ -834,7 +835,7 @@ DerivationType BasicDerivation::type() const throw Error("can't mix derivation output types"); } -Sync drvHashes; +DrvHashes drvHashes; /* pathDerivationModulo and hashDerivationModulo are mutually recursive */ @@ -844,16 +845,13 @@ Sync drvHashes; */ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) { - { - auto hashes = drvHashes.lock(); - auto h = hashes->find(drvPath); - if (h != hashes->end()) { - return h->second; - } + std::optional hash; + if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { + return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); // Cache it - drvHashes.lock()->insert_or_assign(drvPath, h); + drvHashes.insert_or_assign(drvPath, h); return h; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 385215fe0..dd33f5f84 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,7 @@ #include "nix/store/derivations.hh" #include "nix/store/globals.hh" #include "nix/store/local-store.hh" +#include "nix/store/path.hh" #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" @@ -13,14 +14,10 @@ # include "nix/util/processes.hh" #endif +#include +#include #include - -#include #include -#include -#include - -#include #include #include #include @@ -314,7 +311,7 @@ Roots LocalStore::findRoots(bool censor) /** * Key is a mere string because cannot has path with macOS's libc++ */ -typedef std::unordered_map> UncheckedRoots; +typedef boost::unordered_flat_map> UncheckedRoots; static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) { @@ -463,13 +460,13 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; - std::unordered_set roots, dead, alive; + boost::unordered_flat_set> roots, dead, alive; struct Shared { // The temp roots only store the hash part to make it easier to // ignore suffixes like '.lock', '.chroot' and '.check'. - std::unordered_set tempRoots; + boost::unordered_flat_set tempRoots; // Hash part of the store path currently being deleted, if // any. @@ -672,7 +669,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - std::unordered_map referrersCache; + boost::unordered_flat_map> referrersCache; /* Helper function that visits all paths reachable from `start` via the referrers edges and optionally derivers and derivation diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 18479b425..08bb7183f 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -11,7 +11,7 @@ #include "nix/util/sync.hh" #include "nix/util/variant-wrapper.hh" -#include +#include #include namespace nix { @@ -507,13 +507,23 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut */ std::map staticOutputHashes(Store & store, const Derivation & drv); +struct DrvHashFct +{ + using is_avalanching = std::true_type; + + std::size_t operator()(const StorePath & path) const noexcept + { + return std::hash{}(path.to_string()); + } +}; + /** * Memoisation of hashDerivationModulo(). */ -typedef std::map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. -extern Sync drvHashes; +extern DrvHashes drvHashes; struct Source; struct Sink; diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 9f2255025..fba9d6079 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -1,13 +1,13 @@ #pragma once ///@file -#include - #include "nix/store/store-api.hh" +#include +#include namespace nix { -typedef std::unordered_map> Roots; +typedef boost::unordered_flat_map, std::hash> Roots; struct GCOptions { diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 1184be8ed..b871aaee2 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -11,7 +11,7 @@ #include #include #include -#include +#include namespace nix { @@ -442,7 +442,7 @@ private: std::pair createTempDirInStore(); - typedef std::unordered_set InodeHash; + typedef boost::unordered_flat_set InodeHash; InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 8de41fe19..c5e1747c1 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,5 +1,3 @@ -#include - #include "nix/store/derivations.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" @@ -13,6 +11,8 @@ #include "nix/store/filetransfer.hh" #include "nix/util/strings.hh" +#include + namespace nix { void Store::computeFSClosure( @@ -106,7 +106,7 @@ MissingPaths Store::queryMissing(const std::vector & targets) struct State { - std::unordered_set done; + boost::unordered_flat_set done; MissingPaths res; }; diff --git a/src/libutil/include/nix/util/canon-path.hh b/src/libutil/include/nix/util/canon-path.hh index cb8b4325d..334c9e332 100644 --- a/src/libutil/include/nix/util/canon-path.hh +++ b/src/libutil/include/nix/util/canon-path.hh @@ -258,7 +258,7 @@ public: */ std::string makeRelative(const CanonPath & path) const; - friend class std::hash; + friend struct std::hash; }; std::ostream & operator<<(std::ostream & stream, const CanonPath & path); @@ -268,6 +268,8 @@ std::ostream & operator<<(std::ostream & stream, const CanonPath & path); template<> struct std::hash { + using is_avalanching = std::true_type; + std::size_t operator()(const nix::CanonPath & s) const noexcept { return std::hash{}(s.path); diff --git a/src/libutil/linux/cgroup.cc b/src/libutil/linux/cgroup.cc index 20d19ae7d..9e78ac6d2 100644 --- a/src/libutil/linux/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -4,10 +4,10 @@ #include "nix/util/file-system.hh" #include "nix/util/finally.hh" +#include #include #include #include -#include #include #include @@ -76,7 +76,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu int round = 1; - std::unordered_set pidsShown; + boost::unordered_flat_set pidsShown; while (true) { auto pids = tokenizeString>(readFile(procsFile)); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index b932f6ab5..877c63331 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -104,7 +104,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa if (cache.size() >= 16384) cache.clear(); - cache.emplace(absPath, st); + cache.emplace(std::move(absPath), st); return st; } diff --git a/src/nix/env.cc b/src/nix/env.cc index d91ee72d7..c8fb5bee0 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -71,7 +71,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); - std::unordered_set done; + boost::unordered_flat_set> done; std::queue todo; for (auto & path : outPaths) todo.push(path); From 9dbc2cae4f70f2243fedc618fe90b132a67d6441 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sun, 7 Sep 2025 14:16:32 +0200 Subject: [PATCH 3/3] hashmaps with string keys: add transparent lookups --- src/libexpr/include/nix/expr/eval.hh | 9 +++++---- src/libstore/gc.cc | 19 ++++++++++++------- src/libstore/include/nix/store/gc-store.hh | 6 +++++- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 75ed12664..f1cead47b 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -474,7 +474,8 @@ private: LookupPath lookupPath; - boost::unordered_flat_map> lookupPathResolved; + boost::unordered_flat_map, StringViewHash, std::equal_to<>> + lookupPathResolved; /** * Cache used by prim_match(). @@ -751,8 +752,8 @@ public: boost::unordered_flat_map< std::string, Value *, - std::hash, - std::equal_to, + StringViewHash, + std::equal_to<>, traceable_allocator>> internalPrimOps; @@ -1019,7 +1020,7 @@ private: bool countCalls; - typedef boost::unordered_flat_map PrimOpCalls; + typedef boost::unordered_flat_map> PrimOpCalls; PrimOpCalls primOpCalls; typedef boost::unordered_flat_map FunctionCalls; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index dd33f5f84..fdbc670df 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -311,7 +311,12 @@ Roots LocalStore::findRoots(bool censor) /** * Key is a mere string because cannot has path with macOS's libc++ */ -typedef boost::unordered_flat_map> UncheckedRoots; +typedef boost::unordered_flat_map< + std::string, + boost::unordered_flat_set>, + StringViewHash, + std::equal_to<>> + UncheckedRoots; static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) { @@ -325,7 +330,7 @@ static void readProcLink(const std::filesystem::path & file, UncheckedRoots & ro throw; } if (buf.is_absolute()) - roots[buf.string()].emplace(file.string()); + roots[buf].emplace(file.string()); } static std::string quoteRegexChars(const std::string & raw) @@ -466,7 +471,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { // The temp roots only store the hash part to make it easier to // ignore suffixes like '.lock', '.chroot' and '.check'. - boost::unordered_flat_set tempRoots; + boost::unordered_flat_set> tempRoots; // Hash part of the store path currently being deleted, if // any. @@ -575,9 +580,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto storePath = maybeParseStorePath(path); if (storePath) { debug("got new GC root '%s'", path); - auto hashPart = std::string(storePath->hashPart()); + auto hashPart = storePath->hashPart(); auto shared(_shared.lock()); - shared->tempRoots.insert(hashPart); + shared->tempRoots.emplace(hashPart); /* If this path is currently being deleted, then we have to wait until deletion is finished to ensure that @@ -629,7 +634,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Roots tempRoots; findTempRoots(tempRoots, true); for (auto & root : tempRoots) { - _shared.lock()->tempRoots.insert(std::string(root.first.hashPart())); + _shared.lock()->tempRoots.emplace(root.first.hashPart()); roots.insert(root.first); } @@ -736,7 +741,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return; { - auto hashPart = std::string(path->hashPart()); + auto hashPart = path->hashPart(); auto shared(_shared.lock()); if (shared->tempRoots.count(hashPart)) { debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index fba9d6079..5a4a6db14 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -7,7 +7,11 @@ namespace nix { -typedef boost::unordered_flat_map, std::hash> Roots; +typedef boost::unordered_flat_map< + StorePath, + boost::unordered_flat_set>, + std::hash> + Roots; struct GCOptions {