nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-08 19:46:02 +01:00
Code Activity

ci: Collect code coverage in tests

Browse source 
This adds the necessary infrastructure to collect
code coverage in CI, which could be useful to look
at munually or track consistently via something like
codecov.

Co-authored-by: Jade Lovelace <lix@jade.fyi>
This commit is contained in:
Sergei Zimmerman 2025-08-04 23:50:02 +03:00
parent 0889960869
commit 4333a9d5a8
No known key found for this signature in database
6 changed files with 200 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

30
.github/workflows/ci.yml vendored
View file

@ -29,18 +29,21 @@ jobs:
- scenario: on ubuntu
runs-on: ubuntu-24.04
os: linux
sanitizers: false
instrumented: false
primary: true
stdenv: stdenv
- scenario: on macos
runs-on: macos-14
os: darwin
sanitizers: false
instrumented: false
primary: true
- scenario: on ubuntu (with sanitizers)
stdenv: stdenv
- scenario: on ubuntu (with sanitizers / coverage)
runs-on: ubuntu-24.04
os: linux
sanitizers: true
instrumented: true
primary: false
stdenv: clangStdenv
name: tests ${{ matrix.scenario }}
runs-on: ${{ matrix.runs-on }}
timeout-minutes: 60
@ -63,13 +66,28 @@ jobs:
if: matrix.os == 'linux'
- name: Run component tests
run: |
nix build --file ci/gha/tests componentTests -L \
--arg withSanitizers ${{ matrix.sanitizers }}
nix build --file ci/gha/tests/wrapper.nix componentTests -L \
--arg withInstrumentation ${{ matrix.instrumented }} \
--argstr stdenv "${{ matrix.stdenv }}"
- name: Run flake checks and prepare the installer tarball
run: |
ci/gha/tests/build-checks
ci/gha/tests/prepare-installer-for-github-actions
if: ${{ matrix.primary }}
- name: Collect code coverage
run: |
nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \
--arg withInstrumentation ${{ matrix.instrumented }} \
--argstr stdenv "${{ matrix.stdenv }}" \
--out-link coverage-reports
cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
if: ${{ matrix.instrumented }}
- name: Upload coverage reports
uses: actions/upload-artifact@v4
with:
name: coverage-reports
path: coverage-reports/
if: ${{ matrix.instrumented }}
- name: Upload installer tarball
uses: actions/upload-artifact@v4
with:

176
ci/gha/tests/default.nix
View file

@ -5,15 +5,78 @@
getStdenv ? p: p.stdenv,
componentTestsPrefix ? "",
withSanitizers ? false,
withCoverage ? false,
...
}:
let
inherit (pkgs) lib;
hydraJobs = nixFlake.hydraJobs;
packages' = nixFlake.packages.${system};
stdenv = (getStdenv pkgs);
enableSanitizersLayer = finalAttrs: prevAttrs: {
mesonFlags =
(prevAttrs.mesonFlags or [ ])
++ [
# Run all tests with UBSAN enabled. Running both with ubsan and
# without doesn't seem to have much immediate benefit for doubling
# the GHA CI workaround.
#
# TODO: Work toward enabling "address,undefined" if it seems feasible.
# This would maybe require dropping Boost coroutines and ignoring intentional
# memory leaks with detect_leaks=0.
(lib.mesonOption "b_sanitize" "undefined")
]
++ (lib.optionals stdenv.cc.isClang [
# https://www.github.com/mesonbuild/meson/issues/764
(lib.mesonBool "b_lundef" false)
]);
};
collectCoverageLayer = finalAttrs: prevAttrs: {
env =
let
# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#the-code-coverage-workflow
coverageFlags = [
"-fprofile-instr-generate"
"-fcoverage-mapping"
];
in
{
CFLAGS = toString coverageFlags;
CXXFLAGS = toString coverageFlags;
};
# Done in a pre-configure hook, because $NIX_BUILD_TOP needs to be substituted.
preConfigure =
prevAttrs.preConfigure or ""
+ ''
mappingFlag=" -fcoverage-prefix-map=$NIX_BUILD_TOP/${finalAttrs.src.name}=${finalAttrs.src}"
CFLAGS+="$mappingFlag"
CXXFLAGS+="$mappingFlag"
'';
};
componentOverrides =
(lib.optional withSanitizers enableSanitizersLayer)
++ (lib.optional withCoverage collectCoverageLayer);
in
{
rec {
nixComponents =
(nixFlake.lib.makeComponents {
inherit pkgs;
inherit getStdenv;
}).overrideScope
(
final: prev: {
nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
}
);
/**
Top-level tests for the flake outputs, as they would be built by hydra.
These tests generally can't be overridden to run with sanitizers.
@ -52,33 +115,6 @@ in
};
componentTests =
let
nixComponents =
(nixFlake.lib.makeComponents {
inherit pkgs;
inherit getStdenv;
}).overrideScope
(
final: prev: {
nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
mesonComponentOverrides = finalAttrs: prevAttrs: {
mesonFlags =
(prevAttrs.mesonFlags or [ ])
++ lib.optionals withSanitizers [
# Run all tests with UBSAN enabled. Running both with ubsan and
# without doesn't seem to have much immediate benefit for doubling
# the GHA CI workaround.
#
# TODO: Work toward enabling "address,undefined" if it seems feasible.
# This would maybe require dropping Boost coroutines and ignoring intentional
# memory leaks with detect_leaks=0.
(lib.mesonOption "b_sanitize" "undefined")
];
};
}
);
in
(lib.concatMapAttrs (
pkgName: pkg:
lib.concatMapAttrs (testName: test: {
@ -88,4 +124,88 @@ in
// lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) {
"${componentTestsPrefix}nix-functional-tests" = nixComponents.nix-functional-tests;
};
codeCoverage =
let
componentsTestsToProfile =
(builtins.mapAttrs (n: v: nixComponents.${n}.tests.run) {
"nix-util-tests" = { };
"nix-store-tests" = { };
"nix-fetchers-tests" = { };
"nix-expr-tests" = { };
"nix-flake-tests" = { };
})
// {
inherit (nixComponents) nix-functional-tests;
};
coverageProfileDrvs = lib.mapAttrs (
n: v:
v.overrideAttrs (
finalAttrs: prevAttrs: {
outputs = (prevAttrs.outputs or [ "out" ]) ++ [ "profraw" ];
env = {
LLVM_PROFILE_FILE = "${placeholder "profraw"}/%m";
};
}
)
) componentsTestsToProfile;
coverageProfiles = lib.mapAttrsToList (n: v: lib.getOutput "profraw" v) coverageProfileDrvs;
mergedProfdata =
pkgs.runCommand "merged-profdata"
{
__structuredAttrs = true;
nativeBuildInputs = [ pkgs.llvmPackages.libllvm ];
inherit coverageProfiles;
}
''
rawProfiles=()
for dir in "''\${coverageProfiles[@]}"; do
rawProfiles+=($dir/*)
done
llvm-profdata merge -sparse -output $out "''\${rawProfiles[@]}"
'';
coverageReports =
let
nixComponentDrvs = lib.filter (lib.isDerivation) (lib.attrValues nixComponents);
in
pkgs.runCommand "code-coverage-report"
{
nativeBuildInputs = [
pkgs.llvmPackages.libllvm
];
__structuredAttrs = true;
nixComponents = nixComponentDrvs;
}
''
# ${toString (lib.map (v: v.src) nixComponentDrvs)}
binaryFiles=()
for dir in "''\${nixComponents[@]}"; do
readarray -t filesInDir < <(find "$dir" -type f -executable)
binaryFiles+=("''\${filesInDir[@]}")
done
arguments=$(concatStringsSep " -object " binaryFiles)
llvm-cov show $arguments -instr-profile ${mergedProfdata} -output-dir $out -format=html
{
echo "# Code coverage summary (generated via \`llvm-cov\`):"
echo
echo '```'
llvm-cov report $arguments -instr-profile ${mergedProfdata} -format=text -use-color=false
echo '```'
echo
} >> $out/index.txt
'';
in
assert withCoverage;
assert stdenv.cc.isClang;
{
inherit coverageProfileDrvs mergedProfdata coverageReports;
};
}

16
ci/gha/tests/wrapper.nix Normal file
View file

@ -0,0 +1,16 @@
{
nixFlake ? builtins.getFlake ("git+file://" + toString ../../..),
system ? builtins.currentSystem,
pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system},
stdenv ? "stdenv",
componentTestsPrefix ? "",
withInstrumentation ? false,
}@args:
import ./. (
args
// {
getStdenv = p: p.${stdenv};
withSanitizers = withInstrumentation;
withCoverage = withInstrumentation;
}
)

8
nix-meson-build-support/common/meson.build
View file

@ -32,3 +32,11 @@ do_pch = cxx.get_id() == 'clang'
if cxx.get_id() == 'clang'
add_project_arguments('-fpch-instantiate-templates', language : 'cpp')
endif
# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
# passed when building shared libs, at least on Linux
if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
'b_sanitize',
))
add_project_link_arguments('-shared-libasan', language : 'cpp')
endif

2
tests/functional/flakes/run.sh
View file

@ -41,11 +41,13 @@ nix run -f shell-hello.nix env > $TEST_ROOT/actual-env
# - we unset TMPDIR on macOS if it contains /var/folders. bad. https://github.com/NixOS/nix/issues/7731
# - _ is set by bash and is expected to differ because it contains the original command
# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control
# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection
sed -i \
-e 's/PATH=.*/PATH=.../' \
-e 's/_=.*/_=.../' \
-e '/^TMPDIR=\/var\/folders\/.*/d' \
-e '/^__CF_USER_TEXT_ENCODING=.*$/d' \
-e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \
$TEST_ROOT/expected-env $TEST_ROOT/actual-env
sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted
# nix run appears to clear _. I don't understand why. Is this ok?

2
tests/functional/shell.sh
View file

@ -34,11 +34,13 @@ nix shell -f shell-hello.nix hello -c env > "$TEST_ROOT/actual-env"
# - we unset TMPDIR on macOS if it contains /var/folders
# - _ is set by bash and is expectedf to differ because it contains the original command
# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control
# - __LLVM_PROFILE_RT_INIT_ONCE - implementation detail of LLVM source code coverage collection
sed -i \
-e 's/PATH=.*/PATH=.../' \
-e 's/_=.*/_=.../' \
-e '/^TMPDIR=\/var\/folders\/.*/d' \
-e '/^__CF_USER_TEXT_ENCODING=.*$/d' \
-e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \
"$TEST_ROOT/expected-env" "$TEST_ROOT/actual-env"
sort "$TEST_ROOT/expected-env" > "$TEST_ROOT/expected-env.sorted"
sort "$TEST_ROOT/actual-env" > "$TEST_ROOT/actual-env.sorted"