Merge pull request #13686 from xokdvium/ci-coverage

ci: Collect code coverage in tests

.github/workflows/ci.yml

@@ -29,18 +29,21 @@ jobs:
           - scenario: on ubuntu
             runs-on: ubuntu-24.04
             os: linux
-            sanitizers: false
+            instrumented: false
             primary: true
+            stdenv: stdenv
           - scenario: on macos
             runs-on: macos-14
             os: darwin
-            sanitizers: false
+            instrumented: false
             primary: true
-          - scenario: on ubuntu (with sanitizers)
+            stdenv: stdenv
+          - scenario: on ubuntu (with sanitizers / coverage)
             runs-on: ubuntu-24.04
             os: linux
-            sanitizers: true
+            instrumented: true
             primary: false
+            stdenv: clangStdenv
     name: tests ${{ matrix.scenario }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
@@ -63,13 +66,28 @@ jobs:
       if: matrix.os == 'linux'
     - name: Run component tests
       run: |
-        nix build --file ci/gha/tests componentTests -L \
+        nix build --file ci/gha/tests/wrapper.nix componentTests -L \
-          --arg withSanitizers ${{ matrix.sanitizers }}
+          --arg withInstrumentation ${{ matrix.instrumented }} \
+          --argstr stdenv "${{ matrix.stdenv }}"
     - name: Run flake checks and prepare the installer tarball
       run: |
         ci/gha/tests/build-checks
         ci/gha/tests/prepare-installer-for-github-actions
       if: ${{ matrix.primary }}
+    - name: Collect code coverage
+      run: |
+        nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \
+          --arg withInstrumentation ${{ matrix.instrumented }} \
+          --argstr stdenv "${{ matrix.stdenv }}" \
+          --out-link coverage-reports
+        cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
+      if: ${{ matrix.instrumented }}
+    - name: Upload coverage reports
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-reports
+        path: coverage-reports/
+      if: ${{ matrix.instrumented }}
     - name: Upload installer tarball
       uses: actions/upload-artifact@v4
       with:

ci/gha/tests/default.nix

@@ -5,15 +5,78 @@
   getStdenv ? p: p.stdenv,
   componentTestsPrefix ? "",
   withSanitizers ? false,
+  withCoverage ? false,
+  ...
 }:
 
 let
   inherit (pkgs) lib;
   hydraJobs = nixFlake.hydraJobs;
   packages' = nixFlake.packages.${system};
+  stdenv = (getStdenv pkgs);
+
+  enableSanitizersLayer = finalAttrs: prevAttrs: {
+    mesonFlags =
+      (prevAttrs.mesonFlags or [ ])
+      ++ [
+        # Run all tests with UBSAN enabled. Running both with ubsan and
+        # without doesn't seem to have much immediate benefit for doubling
+        # the GHA CI workaround.
+        #
+        # TODO: Work toward enabling "address,undefined" if it seems feasible.
+        # This would maybe require dropping Boost coroutines and ignoring intentional
+        # memory leaks with detect_leaks=0.
+        (lib.mesonOption "b_sanitize" "undefined")
+      ]
+      ++ (lib.optionals stdenv.cc.isClang [
+        # https://www.github.com/mesonbuild/meson/issues/764
+        (lib.mesonBool "b_lundef" false)
+      ]);
+  };
+
+  collectCoverageLayer = finalAttrs: prevAttrs: {
+    env =
+      let
+        # https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#the-code-coverage-workflow
+        coverageFlags = [
+          "-fprofile-instr-generate"
+          "-fcoverage-mapping"
+        ];
+      in
+      {
+        CFLAGS = toString coverageFlags;
+        CXXFLAGS = toString coverageFlags;
+      };
+
+    # Done in a pre-configure hook, because $NIX_BUILD_TOP needs to be substituted.
+    preConfigure =
+      prevAttrs.preConfigure or ""
+      + ''
+        mappingFlag=" -fcoverage-prefix-map=$NIX_BUILD_TOP/${finalAttrs.src.name}=${finalAttrs.src}"
+        CFLAGS+="$mappingFlag"
+        CXXFLAGS+="$mappingFlag"
+      '';
+  };
+
+  componentOverrides =
+    (lib.optional withSanitizers enableSanitizersLayer)
+    ++ (lib.optional withCoverage collectCoverageLayer);
 in
 
-{
+rec {
+  nixComponents =
+    (nixFlake.lib.makeComponents {
+      inherit pkgs;
+      inherit getStdenv;
+    }).overrideScope
+      (
+        final: prev: {
+          nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
+
+          mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
+        }
+      );
+
   /**
     Top-level tests for the flake outputs, as they would be built by hydra.
     These tests generally can't be overridden to run with sanitizers.
@@ -52,33 +115,6 @@ in
   };
 
   componentTests =
-    let
-      nixComponents =
-        (nixFlake.lib.makeComponents {
-          inherit pkgs;
-          inherit getStdenv;
-        }).overrideScope
-          (
-            final: prev: {
-              nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
-
-              mesonComponentOverrides = finalAttrs: prevAttrs: {
-                mesonFlags =
-                  (prevAttrs.mesonFlags or [ ])
-                  ++ lib.optionals withSanitizers [
-                    # Run all tests with UBSAN enabled. Running both with ubsan and
-                    # without doesn't seem to have much immediate benefit for doubling
-                    # the GHA CI workaround.
-                    #
-                    # TODO: Work toward enabling "address,undefined" if it seems feasible.
-                    # This would maybe require dropping Boost coroutines and ignoring intentional
-                    # memory leaks with detect_leaks=0.
-                    (lib.mesonOption "b_sanitize" "undefined")
-                  ];
-              };
-            }
-          );
-    in
     (lib.concatMapAttrs (
       pkgName: pkg:
       lib.concatMapAttrs (testName: test: {
@@ -88,4 +124,88 @@ in
     // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) {
       "${componentTestsPrefix}nix-functional-tests" = nixComponents.nix-functional-tests;
     };
+
+  codeCoverage =
+    let
+      componentsTestsToProfile =
+        (builtins.mapAttrs (n: v: nixComponents.${n}.tests.run) {
+          "nix-util-tests" = { };
+          "nix-store-tests" = { };
+          "nix-fetchers-tests" = { };
+          "nix-expr-tests" = { };
+          "nix-flake-tests" = { };
+        })
+        // {
+          inherit (nixComponents) nix-functional-tests;
+        };
+
+      coverageProfileDrvs = lib.mapAttrs (
+        n: v:
+        v.overrideAttrs (
+          finalAttrs: prevAttrs: {
+            outputs = (prevAttrs.outputs or [ "out" ]) ++ [ "profraw" ];
+            env = {
+              LLVM_PROFILE_FILE = "${placeholder "profraw"}/%m";
+            };
+          }
+        )
+      ) componentsTestsToProfile;
+
+      coverageProfiles = lib.mapAttrsToList (n: v: lib.getOutput "profraw" v) coverageProfileDrvs;
+
+      mergedProfdata =
+        pkgs.runCommand "merged-profdata"
+          {
+            __structuredAttrs = true;
+            nativeBuildInputs = [ pkgs.llvmPackages.libllvm ];
+            inherit coverageProfiles;
+          }
+          ''
+            rawProfiles=()
+            for dir in "''\${coverageProfiles[@]}"; do
+              rawProfiles+=($dir/*)
+            done
+            llvm-profdata merge -sparse -output $out "''\${rawProfiles[@]}"
+          '';
+
+      coverageReports =
+        let
+          nixComponentDrvs = lib.filter (lib.isDerivation) (lib.attrValues nixComponents);
+        in
+        pkgs.runCommand "code-coverage-report"
+          {
+            nativeBuildInputs = [
+              pkgs.llvmPackages.libllvm
+            ];
+            __structuredAttrs = true;
+            nixComponents = nixComponentDrvs;
+          }
+          ''
+            # ${toString (lib.map (v: v.src) nixComponentDrvs)}
+
+            binaryFiles=()
+            for dir in "''\${nixComponents[@]}"; do
+              readarray -t filesInDir < <(find "$dir" -type f -executable)
+              binaryFiles+=("''\${filesInDir[@]}")
+            done
+
+            arguments=$(concatStringsSep " -object " binaryFiles)
+            llvm-cov show $arguments -instr-profile ${mergedProfdata} -output-dir $out -format=html
+
+            {
+              echo "# Code coverage summary (generated via \`llvm-cov\`):"
+              echo
+              echo '```'
+              llvm-cov report $arguments -instr-profile ${mergedProfdata} -format=text -use-color=false
+              echo '```'
+              echo
+            } >> $out/index.txt
+
+          '';
+    in
+    assert withCoverage;
+    assert stdenv.cc.isClang;
+    {
+      inherit coverageProfileDrvs mergedProfdata coverageReports;
+    };
 }

ci/gha/tests/wrapper.nix

@@ -0,0 +1,16 @@
+{
+  nixFlake ? builtins.getFlake ("git+file://" + toString ../../..),
+  system ? builtins.currentSystem,
+  pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system},
+  stdenv ? "stdenv",
+  componentTestsPrefix ? "",
+  withInstrumentation ? false,
+}@args:
+import ./. (
+  args
+  // {
+    getStdenv = p: p.${stdenv};
+    withSanitizers = withInstrumentation;
+    withCoverage = withInstrumentation;
+  }
+)

nix-meson-build-support/common/meson.build

@@ -32,3 +32,11 @@ do_pch = cxx.get_id() == 'clang'
 if cxx.get_id() == 'clang'
   add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
 endif
+
+# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
+# passed when building shared libs, at least on Linux
+if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
+  'b_sanitize',
+))
+  add_project_link_arguments('-shared-libasan', language : 'cpp')
+endif

tests/functional/flakes/run.sh

@@ -41,11 +41,13 @@ nix run -f shell-hello.nix env > $TEST_ROOT/actual-env
 # - we unset TMPDIR on macOS if it contains /var/folders. bad. https://github.com/NixOS/nix/issues/7731
 # - _ is set by bash and is expected to differ because it contains the original command
 # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control
+# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection
 sed -i \
   -e 's/PATH=.*/PATH=.../' \
   -e 's/_=.*/_=.../' \
   -e '/^TMPDIR=\/var\/folders\/.*/d' \
   -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \
+  -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \
   $TEST_ROOT/expected-env $TEST_ROOT/actual-env
 sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted
 # nix run appears to clear _. I don't understand why. Is this ok?

tests/functional/shell.sh

@@ -34,11 +34,13 @@ nix shell -f shell-hello.nix hello -c env > "$TEST_ROOT/actual-env"
 # - we unset TMPDIR on macOS if it contains /var/folders
 # - _ is set by bash and is expectedf to differ because it contains the original command
 # - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control
+# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection
 sed -i \
   -e 's/PATH=.*/PATH=.../' \
   -e 's/_=.*/_=.../' \
   -e '/^TMPDIR=\/var\/folders\/.*/d' \
   -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \
+  -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \
   "$TEST_ROOT/expected-env" "$TEST_ROOT/actual-env"
 sort "$TEST_ROOT/expected-env" > "$TEST_ROOT/expected-env.sorted"
 sort "$TEST_ROOT/actual-env" > "$TEST_ROOT/actual-env.sorted"