EvalState: Don't maintain stats by default

These counters are extremely expensive in a multi-threaded
program. For instance, disabling them speeds up evaluation of the
NixOS/nix/2.21.2 from 32.6s to 17.8s.

src/libexpr/eval.cc

@@ -287,6 +287,7 @@ EvalState::EvalState(
     assertGCInitialized();
 
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
+    static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
 
     /* Construct the Nix expression search path. */
     assert(lookupPath.elements.empty());
@@ -892,7 +893,7 @@ Value * EvalState::getBool(bool b)
     return b ? &Value::vTrue : &Value::vFalse;
 }
 
-static std::atomic<uint64_t> nrThunks = 0;
+static Counter nrThunks;
 
 static inline void mkThunk(Value & v, Env & env, Expr * expr)
 {
@@ -2891,11 +2892,11 @@ bool EvalState::fullGC()
 #endif
 }
 
+bool Counter::enabled = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
+
 void EvalState::maybePrintStats()
 {
-    bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
+    if (Counter::enabled) {
-
-    if (showStats) {
         // Make the final heap size more deterministic.
 #if NIX_USE_BOEHMGC
         if (!fullGC()) {
@@ -2944,7 +2945,7 @@ void EvalState::printStatistics()
         {"elements", nrValuesInEnvs.load()},
         {"bytes", bEnvs},
     };
-    topObj["nrExprs"] = Expr::nrExprs;
+    topObj["nrExprs"] = Expr::nrExprs.load();
     topObj["list"] = {
         {"elements", nrListElems.load()},
         {"bytes", bLists},

src/libexpr/include/nix/expr/counter.hh

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+
+namespace nix {
+
+/**
+ * An atomic counter aligned on a cache line to prevent false sharing.
+ * The counter is only enabled when the `NIX_SHOW_STATS` environment
+ * variable is set. This is to prevent contention on these counters
+ * when multi-threaded evaluation is enabled.
+ */
+struct alignas(64) Counter
+{
+    using value_type = uint64_t;
+
+    std::atomic<value_type> inner{0};
+
+    static bool enabled;
+
+    Counter() {}
+
+    operator value_type() const noexcept
+    {
+        return inner;
+    }
+
+    void operator=(value_type n) noexcept
+    {
+        inner = n;
+    }
+
+    value_type load() const noexcept
+    {
+        return inner;
+    }
+
+    value_type operator++() noexcept
+    {
+        return enabled ? ++inner : 0;
+    }
+
+    value_type operator++(int) noexcept
+    {
+        return enabled ? inner++ : 0;
+    }
+
+    value_type operator--() noexcept
+    {
+        return enabled ? --inner : 0;
+    }
+
+    value_type operator--(int) noexcept
+    {
+        return enabled ? inner-- : 0;
+    }
+
+    value_type operator+=(value_type n) noexcept
+    {
+        return enabled ? inner += n : 0;
+    }
+
+    value_type operator-=(value_type n) noexcept
+    {
+        return enabled ? inner -= n : 0;
+    }
+};
+
+} // namespace nix

src/libexpr/include/nix/expr/eval.hh

@@ -16,6 +16,7 @@
 #include "nix/expr/search-path.hh"
 #include "nix/expr/repl-exit-status.hh"
 #include "nix/util/ref.hh"
+#include "nix/expr/counter.hh"
 
 // For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS`
 #include "nix/expr/config.hh"
@@ -961,19 +962,19 @@ private:
      */
     std::string mkSingleDerivedPathStringRaw(const SingleDerivedPath & p);
 
-    std::atomic<uint64_t> nrEnvs = 0;
+    Counter nrEnvs;
-    std::atomic<uint64_t> nrValuesInEnvs = 0;
+    Counter nrValuesInEnvs;
-    std::atomic<uint64_t> nrValues = 0;
+    Counter nrValues;
-    std::atomic<uint64_t> nrListElems = 0;
+    Counter nrListElems;
-    std::atomic<uint64_t> nrLookups = 0;
+    Counter nrLookups;
-    std::atomic<uint64_t> nrAttrsets = 0;
+    Counter nrAttrsets;
-    std::atomic<uint64_t> nrAttrsInAttrsets = 0;
+    Counter nrAttrsInAttrsets;
-    std::atomic<uint64_t> nrAvoided = 0;
+    Counter nrAvoided;
-    std::atomic<uint64_t> nrOpUpdates = 0;
+    Counter nrOpUpdates;
-    std::atomic<uint64_t> nrOpUpdateValuesCopied = 0;
+    Counter nrOpUpdateValuesCopied;
-    std::atomic<uint64_t> nrListConcats = 0;
+    Counter nrListConcats;
-    std::atomic<uint64_t> nrPrimOpCalls = 0;
+    Counter nrPrimOpCalls;
-    std::atomic<uint64_t> nrFunctionCalls = 0;
+    Counter nrFunctionCalls;
 
     bool countCalls;
 

src/libexpr/include/nix/expr/meson.build

@@ -10,6 +10,7 @@ config_pub_h = configure_file(
 headers = [ config_pub_h ] + files(
   'attr-path.hh',
   'attr-set.hh',
+  'counter.hh',
   'eval-cache.hh',
   'eval-error.hh',
   'eval-gc.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -9,6 +9,7 @@
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/eval-error.hh"
 #include "nix/util/pos-idx.hh"
+#include "nix/expr/counter.hh"
 
 namespace nix {
 
@@ -92,7 +93,7 @@ struct Expr
         Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
     };
 
-    static unsigned long nrExprs;
+    static Counter nrExprs;
 
     Expr()
     {

src/libexpr/nixexpr.cc

@@ -11,7 +11,7 @@
 
 namespace nix {
 
-unsigned long Expr::nrExprs = 0;
+Counter Expr::nrExprs;
 
 ExprBlackHole eBlackHole;