Merge branch 'master' into fzakaria/shellcheck-systemd-multi-user

maintainers/flake-module.nix

@@ -106,11 +106,8 @@
             enable = true;
             excludes = [
               # We haven't linted these files yet
-              ''^tests/functional/compute-levels\.sh$''
               ''^tests/functional/db-migration\.sh$''
-              ''^tests/functional/dependencies\.builder0\.sh$''
               ''^tests/functional/dump-db\.sh$''
-              ''^tests/functional/dyn-drv/build-built-drv\.sh$''
               ''^tests/functional/dyn-drv/eval-outputOf\.sh$''
               ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$''
               ''^tests/functional/dyn-drv/recursive-mod-json\.sh$''

src/libexpr/eval.cc

@@ -287,6 +287,7 @@ EvalState::EvalState(
     assertGCInitialized();
 
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
+    static_assert(sizeof(Counter) == 64, "counters must be 64 bytes");
 
     /* Construct the Nix expression search path. */
     assert(lookupPath.elements.empty());
@@ -892,7 +893,7 @@ Value * EvalState::getBool(bool b)
     return b ? &Value::vTrue : &Value::vFalse;
 }
 
-unsigned long nrThunks = 0;
+static Counter nrThunks;
 
 static inline void mkThunk(Value & v, Env & env, Expr * expr)
 {
@@ -2891,11 +2892,11 @@ bool EvalState::fullGC()
 #endif
 }
 
+bool Counter::enabled = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
+
 void EvalState::maybePrintStats()
 {
-    bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
-
-    if (showStats) {
+    if (Counter::enabled) {
         // Make the final heap size more deterministic.
 #if NIX_USE_BOEHMGC
         if (!fullGC()) {
@@ -2940,18 +2941,18 @@ void EvalState::printStatistics()
 #endif
     };
     topObj["envs"] = {
-        {"number", nrEnvs},
-        {"elements", nrValuesInEnvs},
+        {"number", nrEnvs.load()},
+        {"elements", nrValuesInEnvs.load()},
         {"bytes", bEnvs},
     };
-    topObj["nrExprs"] = Expr::nrExprs;
+    topObj["nrExprs"] = Expr::nrExprs.load();
     topObj["list"] = {
-        {"elements", nrListElems},
+        {"elements", nrListElems.load()},
         {"bytes", bLists},
-        {"concats", nrListConcats},
+        {"concats", nrListConcats.load()},
     };
     topObj["values"] = {
-        {"number", nrValues},
+        {"number", nrValues.load()},
         {"bytes", bValues},
     };
     topObj["symbols"] = {
@@ -2959,9 +2960,9 @@ void EvalState::printStatistics()
         {"bytes", symbols.totalSize()},
     };
     topObj["sets"] = {
-        {"number", nrAttrsets},
+        {"number", nrAttrsets.load()},
         {"bytes", bAttrsets},
-        {"elements", nrAttrsInAttrsets},
+        {"elements", nrAttrsInAttrsets.load()},
     };
     topObj["sizes"] = {
         {"Env", sizeof(Env)},
@@ -2969,13 +2970,13 @@ void EvalState::printStatistics()
         {"Bindings", sizeof(Bindings)},
         {"Attr", sizeof(Attr)},
     };
-    topObj["nrOpUpdates"] = nrOpUpdates;
-    topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
-    topObj["nrThunks"] = nrThunks;
-    topObj["nrAvoided"] = nrAvoided;
-    topObj["nrLookups"] = nrLookups;
-    topObj["nrPrimOpCalls"] = nrPrimOpCalls;
-    topObj["nrFunctionCalls"] = nrFunctionCalls;
+    topObj["nrOpUpdates"] = nrOpUpdates.load();
+    topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied.load();
+    topObj["nrThunks"] = nrThunks.load();
+    topObj["nrAvoided"] = nrAvoided.load();
+    topObj["nrLookups"] = nrLookups.load();
+    topObj["nrPrimOpCalls"] = nrPrimOpCalls.load();
+    topObj["nrFunctionCalls"] = nrFunctionCalls.load();
 #if NIX_USE_BOEHMGC
     topObj["gc"] = {
         {"heapSize", heapSize},

src/libexpr/include/nix/expr/counter.hh

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+
+namespace nix {
+
+/**
+ * An atomic counter aligned on a cache line to prevent false sharing.
+ * The counter is only enabled when the `NIX_SHOW_STATS` environment
+ * variable is set. This is to prevent contention on these counters
+ * when multi-threaded evaluation is enabled.
+ */
+struct alignas(64) Counter
+{
+    using value_type = uint64_t;
+
+    std::atomic<value_type> inner{0};
+
+    static bool enabled;
+
+    Counter() {}
+
+    operator value_type() const noexcept
+    {
+        return inner;
+    }
+
+    void operator=(value_type n) noexcept
+    {
+        inner = n;
+    }
+
+    value_type load() const noexcept
+    {
+        return inner;
+    }
+
+    value_type operator++() noexcept
+    {
+        return enabled ? ++inner : 0;
+    }
+
+    value_type operator++(int) noexcept
+    {
+        return enabled ? inner++ : 0;
+    }
+
+    value_type operator--() noexcept
+    {
+        return enabled ? --inner : 0;
+    }
+
+    value_type operator--(int) noexcept
+    {
+        return enabled ? inner-- : 0;
+    }
+
+    value_type operator+=(value_type n) noexcept
+    {
+        return enabled ? inner += n : 0;
+    }
+
+    value_type operator-=(value_type n) noexcept
+    {
+        return enabled ? inner -= n : 0;
+    }
+};
+
+} // namespace nix

src/libexpr/include/nix/expr/eval.hh

@@ -16,6 +16,7 @@
 #include "nix/expr/search-path.hh"
 #include "nix/expr/repl-exit-status.hh"
 #include "nix/util/ref.hh"
+#include "nix/expr/counter.hh"
 
 // For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS`
 #include "nix/expr/config.hh"
@@ -961,19 +962,19 @@ private:
      */
     std::string mkSingleDerivedPathStringRaw(const SingleDerivedPath & p);
 
-    unsigned long nrEnvs = 0;
-    unsigned long nrValuesInEnvs = 0;
-    unsigned long nrValues = 0;
-    unsigned long nrListElems = 0;
-    unsigned long nrLookups = 0;
-    unsigned long nrAttrsets = 0;
-    unsigned long nrAttrsInAttrsets = 0;
-    unsigned long nrAvoided = 0;
-    unsigned long nrOpUpdates = 0;
-    unsigned long nrOpUpdateValuesCopied = 0;
-    unsigned long nrListConcats = 0;
-    unsigned long nrPrimOpCalls = 0;
-    unsigned long nrFunctionCalls = 0;
+    Counter nrEnvs;
+    Counter nrValuesInEnvs;
+    Counter nrValues;
+    Counter nrListElems;
+    Counter nrLookups;
+    Counter nrAttrsets;
+    Counter nrAttrsInAttrsets;
+    Counter nrAvoided;
+    Counter nrOpUpdates;
+    Counter nrOpUpdateValuesCopied;
+    Counter nrListConcats;
+    Counter nrPrimOpCalls;
+    Counter nrFunctionCalls;
 
     bool countCalls;
 

src/libexpr/include/nix/expr/meson.build

@@ -10,6 +10,7 @@ config_pub_h = configure_file(
 headers = [ config_pub_h ] + files(
   'attr-path.hh',
   'attr-set.hh',
+  'counter.hh',
   'eval-cache.hh',
   'eval-error.hh',
   'eval-gc.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -9,6 +9,7 @@
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/eval-error.hh"
 #include "nix/util/pos-idx.hh"
+#include "nix/expr/counter.hh"
 
 namespace nix {
 
@@ -92,7 +93,7 @@ struct Expr
         Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
     };
 
-    static unsigned long nrExprs;
+    static Counter nrExprs;
 
     Expr()
     {

src/libexpr/nixexpr.cc

@@ -11,7 +11,7 @@
 
 namespace nix {
 
-unsigned long Expr::nrExprs = 0;
+Counter Expr::nrExprs;
 
 ExprBlackHole eBlackHole;
 

src/libstore-tests/data/realisation/simple.json

@@ -0,0 +1,6 @@
+{
+  "dependentRealisations": {},
+  "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
+  "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+  "signatures": []
+}

src/libstore-tests/data/realisation/with-dependent-realisations.json

@@ -0,0 +1,8 @@
+{
+  "dependentRealisations": {
+    "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"
+  },
+  "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
+  "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+  "signatures": []
+}

src/libstore-tests/data/realisation/with-signature.json

@@ -0,0 +1,8 @@
+{
+  "dependentRealisations": {},
+  "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
+  "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+  "signatures": [
+    "asdfasdfasdf"
+  ]
+}

src/libstore-tests/meson.build

@@ -74,6 +74,7 @@ sources = files(
   'outputs-spec.cc',
   'path-info.cc',
   'path.cc',
+  'realisation.cc',
   'references.cc',
   's3-binary-cache-store.cc',
   's3.cc',

src/libstore-tests/realisation.cc

@@ -0,0 +1,105 @@
+#include <regex>
+
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+#include <rapidcheck/gtest.h>
+
+#include "nix/store/store-api.hh"
+
+#include "nix/util/tests/characterization.hh"
+#include "nix/store/tests/libstore.hh"
+
+namespace nix {
+
+class RealisationTest : public CharacterizationTest, public LibStoreTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "realisation";
+
+public:
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
+
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+using nlohmann::json;
+
+struct RealisationJsonTest : RealisationTest, ::testing::WithParamInterface<std::pair<std::string_view, Realisation>>
+{};
+
+TEST_P(RealisationJsonTest, from_json)
+{
+    auto [name, expected] = GetParam();
+    readTest(name + ".json", [&](const auto & encoded_) {
+        auto encoded = json::parse(encoded_);
+        Realisation got = static_cast<Realisation>(encoded);
+        ASSERT_EQ(got, expected);
+    });
+}
+
+TEST_P(RealisationJsonTest, to_json)
+{
+    auto [name, value] = GetParam();
+    writeTest(
+        name + ".json",
+        [&]() -> json { return static_cast<json>(value); },
+        [](const auto & file) { return json::parse(readFile(file)); },
+        [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); });
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    RealisationJSON,
+    RealisationJsonTest,
+    ([] {
+        Realisation simple{
+
+            .id =
+                {
+                    .drvHash = Hash::parseExplicitFormatUnprefixed(
+                        "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+                        HashAlgorithm::SHA256,
+                        HashFormat::Base16),
+                    .outputName = "foo",
+                },
+            .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
+        };
+        return ::testing::Values(
+            std::pair{
+                "simple",
+                simple,
+            },
+            std::pair{
+                "with-signature",
+                [&] {
+                    auto r = simple;
+                    // FIXME actually sign properly
+                    r.signatures = {"asdfasdfasdf"};
+                    return r;
+                }()},
+            std::pair{
+                "with-dependent-realisations",
+                [&] {
+                    auto r = simple;
+                    r.dependentRealisations = {{
+                        {
+                            .drvHash = Hash::parseExplicitFormatUnprefixed(
+                                "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+                                HashAlgorithm::SHA256,
+                                HashFormat::Base16),
+                            .outputName = "foo",
+                        },
+                        StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
+                    }};
+                    return r;
+                }(),
+            });
+    }
+
+     ()));
+
+} // namespace nix

tests/functional/dependencies.builder0.sh

@@ -1,16 +1,20 @@
+# shellcheck shell=bash
+# shellcheck disable=SC2154
 [ "${input1: -2}" = /. ]
+# shellcheck disable=SC2154
 [ "${input2: -2}" = /. ]
 
-mkdir $out
-echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
+# shellcheck disable=SC2154
+mkdir "$out"
+echo "$(cat "$input1"/foo)$(cat "$input2"/bar)" > "$out"/foobar
 
-ln -s $input2 $out/reference-to-input-2
+ln -s "$input2" "$out"/reference-to-input-2
 
 # Self-reference.
-ln -s $out $out/self
+ln -s "$out" "$out"/self
 
 # Executable.
-echo program > $out/program
-chmod +x $out/program
+echo program > "$out"/program
+chmod +x "$out"/program
 
 echo FOO

tests/functional/dyn-drv/build-built-drv.sh

@@ -23,4 +23,4 @@ requireDaemonNewerThan "2.30pre20250515"
 
 out2=$(nix build "${drvDep}^out^out" --no-link)
 
-test $out1 == $out2
+test "$out1" == "$out2"