ci: Collect code coverage in tests

This adds the necessary infrastructure to collect
code coverage in CI, which could be useful to look
at munually or track consistently via something like
codecov.

Co-authored-by: Jade Lovelace <lix@jade.fyi>

.github/workflows/ci.yml

@@ -29,18 +29,21 @@ jobs:
           - scenario: on ubuntu
             runs-on: ubuntu-24.04
             os: linux
-            sanitizers: false
+            instrumented: false
             primary: true
+            stdenv: stdenv
           - scenario: on macos
             runs-on: macos-14
             os: darwin
-            sanitizers: false
+            instrumented: false
             primary: true
-          - scenario: on ubuntu (with sanitizers)
+            stdenv: stdenv
+          - scenario: on ubuntu (with sanitizers / coverage)
             runs-on: ubuntu-24.04
             os: linux
-            sanitizers: true
+            instrumented: true
             primary: false
+            stdenv: clangStdenv
     name: tests ${{ matrix.scenario }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
@@ -63,13 +66,28 @@ jobs:
       if: matrix.os == 'linux'
     - name: Run component tests
       run: |
-        nix build --file ci/gha/tests componentTests -L \
+        nix build --file ci/gha/tests/wrapper.nix componentTests -L \
-          --arg withSanitizers ${{ matrix.sanitizers }}
+          --arg withInstrumentation ${{ matrix.instrumented }} \
+          --argstr stdenv "${{ matrix.stdenv }}"
     - name: Run flake checks and prepare the installer tarball
       run: |
         ci/gha/tests/build-checks
         ci/gha/tests/prepare-installer-for-github-actions
       if: ${{ matrix.primary }}
+    - name: Collect code coverage
+      run: |
+        nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \
+          --arg withInstrumentation ${{ matrix.instrumented }} \
+          --argstr stdenv "${{ matrix.stdenv }}" \
+          --out-link coverage-reports
+        cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
+      if: ${{ matrix.instrumented }}
+    - name: Upload coverage reports
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-reports
+        path: coverage-reports/
+      if: ${{ matrix.instrumented }}
     - name: Upload installer tarball
       uses: actions/upload-artifact@v4
       with:

ci/gha/tests/default.nix

@@ -5,15 +5,78 @@
   getStdenv ? p: p.stdenv,
   componentTestsPrefix ? "",
   withSanitizers ? false,
+  withCoverage ? false,
+  ...
 }:
 
 let
   inherit (pkgs) lib;
   hydraJobs = nixFlake.hydraJobs;
   packages' = nixFlake.packages.${system};
+  stdenv = (getStdenv pkgs);
+
+  enableSanitizersLayer = finalAttrs: prevAttrs: {
+    mesonFlags =
+      (prevAttrs.mesonFlags or [ ])
+      ++ [
+        # Run all tests with UBSAN enabled. Running both with ubsan and
+        # without doesn't seem to have much immediate benefit for doubling
+        # the GHA CI workaround.
+        #
+        # TODO: Work toward enabling "address,undefined" if it seems feasible.
+        # This would maybe require dropping Boost coroutines and ignoring intentional
+        # memory leaks with detect_leaks=0.
+        (lib.mesonOption "b_sanitize" "undefined")
+      ]
+      ++ (lib.optionals stdenv.cc.isClang [
+        # https://www.github.com/mesonbuild/meson/issues/764
+        (lib.mesonBool "b_lundef" false)
+      ]);
+  };
+
+  collectCoverageLayer = finalAttrs: prevAttrs: {
+    env =
+      let
+        # https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#the-code-coverage-workflow
+        coverageFlags = [
+          "-fprofile-instr-generate"
+          "-fcoverage-mapping"
+        ];
+      in
+      {
+        CFLAGS = toString coverageFlags;
+        CXXFLAGS = toString coverageFlags;
+      };
+
+    # Done in a pre-configure hook, because $NIX_BUILD_TOP needs to be substituted.
+    preConfigure =
+      prevAttrs.preConfigure or ""
+      + ''
+        mappingFlag=" -fcoverage-prefix-map=$NIX_BUILD_TOP/${finalAttrs.src.name}=${finalAttrs.src}"
+        CFLAGS+="$mappingFlag"
+        CXXFLAGS+="$mappingFlag"
+      '';
+  };
+
+  componentOverrides =
+    (lib.optional withSanitizers enableSanitizersLayer)
+    ++ (lib.optional withCoverage collectCoverageLayer);
 in
 
-{
+rec {
+  nixComponents =
+    (nixFlake.lib.makeComponents {
+      inherit pkgs;
+      inherit getStdenv;
+    }).overrideScope
+      (
+        final: prev: {
+          nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
+
+          mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
+        }
+      );
+
   /**
     Top-level tests for the flake outputs, as they would be built by hydra.
     These tests generally can't be overridden to run with sanitizers.
@@ -52,33 +115,6 @@ in
   };
 
   componentTests =
-    let
-      nixComponents =
-        (nixFlake.lib.makeComponents {
-          inherit pkgs;
-          inherit getStdenv;
-        }).overrideScope
-          (
-            final: prev: {
-              nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
-
-              mesonComponentOverrides = finalAttrs: prevAttrs: {
-                mesonFlags =
-                  (prevAttrs.mesonFlags or [ ])
-                  ++ lib.optionals withSanitizers [
-                    # Run all tests with UBSAN enabled. Running both with ubsan and
-                    # without doesn't seem to have much immediate benefit for doubling
-                    # the GHA CI workaround.
-                    #
-                    # TODO: Work toward enabling "address,undefined" if it seems feasible.
-                    # This would maybe require dropping Boost coroutines and ignoring intentional
-                    # memory leaks with detect_leaks=0.
-                    (lib.mesonOption "b_sanitize" "undefined")
-                  ];
-              };
-            }
-          );
-    in
     (lib.concatMapAttrs (
       pkgName: pkg:
       lib.concatMapAttrs (testName: test: {
@@ -88,4 +124,88 @@ in
     // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) {
       "${componentTestsPrefix}nix-functional-tests" = nixComponents.nix-functional-tests;
     };
+
+  codeCoverage =
+    let
+      componentsTestsToProfile =
+        (builtins.mapAttrs (n: v: nixComponents.${n}.tests.run) {
+          "nix-util-tests" = { };
+          "nix-store-tests" = { };
+          "nix-fetchers-tests" = { };
+          "nix-expr-tests" = { };
+          "nix-flake-tests" = { };
+        })
+        // {
+          inherit (nixComponents) nix-functional-tests;
+        };
+
+      coverageProfileDrvs = lib.mapAttrs (
+        n: v:
+        v.overrideAttrs (
+          finalAttrs: prevAttrs: {
+            outputs = (prevAttrs.outputs or [ "out" ]) ++ [ "profraw" ];
+            env = {
+              LLVM_PROFILE_FILE = "${placeholder "profraw"}/%m";
+            };
+          }
+        )
+      ) componentsTestsToProfile;
+
+      coverageProfiles = lib.mapAttrsToList (n: v: lib.getOutput "profraw" v) coverageProfileDrvs;
+
+      mergedProfdata =
+        pkgs.runCommand "merged-profdata"
+          {
+            __structuredAttrs = true;
+            nativeBuildInputs = [ pkgs.llvmPackages.libllvm ];
+            inherit coverageProfiles;
+          }
+          ''
+            rawProfiles=()
+            for dir in "''\${coverageProfiles[@]}"; do
+              rawProfiles+=($dir/*)
+            done
+            llvm-profdata merge -sparse -output $out "''\${rawProfiles[@]}"
+          '';
+
+      coverageReports =
+        let
+          nixComponentDrvs = lib.filter (lib.isDerivation) (lib.attrValues nixComponents);
+        in
+        pkgs.runCommand "code-coverage-report"
+          {
+            nativeBuildInputs = [
+              pkgs.llvmPackages.libllvm
+            ];
+            __structuredAttrs = true;
+            nixComponents = nixComponentDrvs;
+          }
+          ''
+            # ${toString (lib.map (v: v.src) nixComponentDrvs)}
+
+            binaryFiles=()
+            for dir in "''\${nixComponents[@]}"; do
+              readarray -t filesInDir < <(find "$dir" -type f -executable)
+              binaryFiles+=("''\${filesInDir[@]}")
+            done
+
+            arguments=$(concatStringsSep " -object " binaryFiles)
+            llvm-cov show $arguments -instr-profile ${mergedProfdata} -output-dir $out -format=html
+
+            {
+              echo "# Code coverage summary (generated via \`llvm-cov\`):"
+              echo
+              echo '```'
+              llvm-cov report $arguments -instr-profile ${mergedProfdata} -format=text -use-color=false
+              echo '```'
+              echo
+            } >> $out/index.txt
+
+          '';
+    in
+    assert withCoverage;
+    assert stdenv.cc.isClang;
+    {
+      inherit coverageProfileDrvs mergedProfdata coverageReports;
+    };
 }

ci/gha/tests/wrapper.nix

@@ -0,0 +1,16 @@
+{
+  nixFlake ? builtins.getFlake ("git+file://" + toString ../../..),
+  system ? builtins.currentSystem,
+  pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system},
+  stdenv ? "stdenv",
+  componentTestsPrefix ? "",
+  withInstrumentation ? false,
+}@args:
+import ./. (
+  args
+  // {
+    getStdenv = p: p.${stdenv};
+    withSanitizers = withInstrumentation;
+    withCoverage = withInstrumentation;
+  }
+)

nix-meson-build-support/common/meson.build

@@ -32,3 +32,11 @@ do_pch = cxx.get_id() == 'clang'
 if cxx.get_id() == 'clang'
   add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
 endif
+
+# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
+# passed when building shared libs, at least on Linux
+if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
+  'b_sanitize',
+))
+  add_project_link_arguments('-shared-libasan', language : 'cpp')
+endif

tests/functional/flakes/run.sh

@@ -41,11 +41,13 @@ nix run -f shell-hello.nix env > $TEST_ROOT/actual-env
 # - we unset TMPDIR on macOS if it contains /var/folders. bad. https://github.com/NixOS/nix/issues/7731
 # - _ is set by bash and is expected to differ because it contains the original command
 # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control
+# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection
 sed -i \
   -e 's/PATH=.*/PATH=.../' \
   -e 's/_=.*/_=.../' \
   -e '/^TMPDIR=\/var\/folders\/.*/d' \
   -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \
+  -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \
   $TEST_ROOT/expected-env $TEST_ROOT/actual-env
 sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted
 # nix run appears to clear _. I don't understand why. Is this ok?

tests/functional/shell.sh

@@ -34,11 +34,13 @@ nix shell -f shell-hello.nix hello -c env > "$TEST_ROOT/actual-env"
 # - we unset TMPDIR on macOS if it contains /var/folders
 # - _ is set by bash and is expectedf to differ because it contains the original command
 # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control
+# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection
 sed -i \
   -e 's/PATH=.*/PATH=.../' \
   -e 's/_=.*/_=.../' \
   -e '/^TMPDIR=\/var\/folders\/.*/d' \
   -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \
+  -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \
   "$TEST_ROOT/expected-env" "$TEST_ROOT/actual-env"
 sort "$TEST_ROOT/expected-env" > "$TEST_ROOT/expected-env.sorted"
 sort "$TEST_ROOT/actual-env" > "$TEST_ROOT/actual-env.sorted"