mirror of
https://github.com/NixOS/nix.git
synced 2025-11-08 19:46:02 +01:00
a400ea4257
This consolidates the separate vm_tests job into the main tests job, simplifying the CI workflow. VM tests now run as part of the regular test matrix.
260 lines
9.1 KiB
Nix
260 lines
9.1 KiB
Nix
{
|
|
nixFlake ? builtins.getFlake ("git+file://" + toString ../../..),
|
|
system ? builtins.currentSystem,
|
|
pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system},
|
|
nixComponents ? (
|
|
nixFlake.lib.makeComponents {
|
|
inherit pkgs;
|
|
inherit getStdenv;
|
|
}
|
|
),
|
|
getStdenv ? p: p.stdenv,
|
|
componentTestsPrefix ? "",
|
|
withSanitizers ? false,
|
|
withCoverage ? false,
|
|
withAWS ? null,
|
|
withCurlS3 ? null,
|
|
...
|
|
}:
|
|
|
|
let
|
|
inherit (pkgs) lib;
|
|
hydraJobs = nixFlake.hydraJobs;
|
|
packages' = nixFlake.packages.${system};
|
|
stdenv = (getStdenv pkgs);
|
|
|
|
enableSanitizersLayer = finalAttrs: prevAttrs: {
|
|
mesonFlags =
|
|
(prevAttrs.mesonFlags or [ ])
|
|
++ [ (lib.mesonOption "b_sanitize" "address,undefined") ]
|
|
++ (lib.optionals stdenv.cc.isClang [
|
|
# https://www.github.com/mesonbuild/meson/issues/764
|
|
(lib.mesonBool "b_lundef" false)
|
|
]);
|
|
};
|
|
|
|
collectCoverageLayer = finalAttrs: prevAttrs: {
|
|
env =
|
|
let
|
|
# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#the-code-coverage-workflow
|
|
coverageFlags = [
|
|
"-fprofile-instr-generate"
|
|
"-fcoverage-mapping"
|
|
];
|
|
in
|
|
{
|
|
CFLAGS = toString coverageFlags;
|
|
CXXFLAGS = toString coverageFlags;
|
|
};
|
|
|
|
# Done in a pre-configure hook, because $NIX_BUILD_TOP needs to be substituted.
|
|
preConfigure = prevAttrs.preConfigure or "" + ''
|
|
mappingFlag=" -fcoverage-prefix-map=$NIX_BUILD_TOP/${finalAttrs.src.name}=${finalAttrs.src}"
|
|
CFLAGS+="$mappingFlag"
|
|
CXXFLAGS+="$mappingFlag"
|
|
'';
|
|
};
|
|
|
|
componentOverrides =
|
|
(lib.optional withSanitizers enableSanitizersLayer)
|
|
++ (lib.optional withCoverage collectCoverageLayer);
|
|
in
|
|
|
|
rec {
|
|
nixComponentsInstrumented = nixComponents.overrideScope (
|
|
final: prev: {
|
|
nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; };
|
|
# Boehm is incompatible with ASAN.
|
|
nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; };
|
|
|
|
# Override AWS configuration if specified
|
|
nix-store = prev.nix-store.override (
|
|
lib.optionalAttrs (withAWS != null) { inherit withAWS; }
|
|
// lib.optionalAttrs (withCurlS3 != null) { inherit withCurlS3; }
|
|
);
|
|
|
|
mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
|
|
# Unclear how to make Perl bindings work with a dynamically linked ASAN.
|
|
nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings;
|
|
}
|
|
);
|
|
|
|
# Import NixOS tests using the instrumented components
|
|
nixosTests = import ../../../tests/nixos {
|
|
inherit lib pkgs;
|
|
nixComponents = nixComponentsInstrumented;
|
|
nixpkgs = nixFlake.inputs.nixpkgs;
|
|
inherit (nixFlake.inputs) nixpkgs-23-11;
|
|
};
|
|
|
|
/**
|
|
Top-level tests for the flake outputs, as they would be built by hydra.
|
|
These tests generally can't be overridden to run with sanitizers.
|
|
*/
|
|
topLevel = {
|
|
installerScriptForGHA = hydraJobs.installerScriptForGHA.${system};
|
|
installTests = hydraJobs.installTests.${system};
|
|
nixpkgsLibTests = hydraJobs.tests.nixpkgsLibTests.${system};
|
|
rl-next = pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
|
|
LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${../../../doc/manual/rl-next} >$out
|
|
'';
|
|
repl-completion = pkgs.callPackage ../../../tests/repl-completion.nix { inherit (packages') nix; };
|
|
|
|
/**
|
|
Checks for our packaging expressions.
|
|
This shouldn't build anything significant; just check that things
|
|
(including derivations) are _set up_ correctly.
|
|
*/
|
|
packaging-overriding =
|
|
let
|
|
nix = packages'.nix;
|
|
in
|
|
assert (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src.patches == [ pkgs.emptyFile ];
|
|
if pkgs.stdenv.buildPlatform.isDarwin then
|
|
lib.warn "packaging-overriding check currently disabled because of a permissions issue on macOS" pkgs.emptyFile
|
|
else
|
|
# If this fails, something might be wrong with how we've wired the scope,
|
|
# or something could be broken in Nixpkgs.
|
|
pkgs.testers.testEqualContents {
|
|
assertion = "trivial patch does not change source contents";
|
|
expected = "${../../..}";
|
|
actual =
|
|
# Same for all components; nix-util is an arbitrary pick
|
|
(nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src;
|
|
};
|
|
};
|
|
|
|
componentTests =
|
|
(lib.concatMapAttrs (
|
|
pkgName: pkg:
|
|
lib.concatMapAttrs (testName: test: {
|
|
"${componentTestsPrefix}${pkgName}-${testName}" = test;
|
|
}) (pkg.tests or { })
|
|
) nixComponentsInstrumented)
|
|
// lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) {
|
|
"${componentTestsPrefix}nix-functional-tests" = nixComponentsInstrumented.nix-functional-tests;
|
|
};
|
|
|
|
codeCoverage =
|
|
let
|
|
componentsTestsToProfile =
|
|
(builtins.mapAttrs (n: v: nixComponentsInstrumented.${n}.tests.run) {
|
|
"nix-util-tests" = { };
|
|
"nix-store-tests" = { };
|
|
"nix-fetchers-tests" = { };
|
|
"nix-expr-tests" = { };
|
|
"nix-flake-tests" = { };
|
|
})
|
|
// {
|
|
inherit (nixComponentsInstrumented) nix-functional-tests;
|
|
};
|
|
|
|
coverageProfileDrvs = lib.mapAttrs (
|
|
n: v:
|
|
v.overrideAttrs (
|
|
finalAttrs: prevAttrs: {
|
|
outputs = (prevAttrs.outputs or [ "out" ]) ++ [ "profraw" ];
|
|
env = {
|
|
LLVM_PROFILE_FILE = "${placeholder "profraw"}/%m";
|
|
};
|
|
}
|
|
)
|
|
) componentsTestsToProfile;
|
|
|
|
coverageProfiles = lib.mapAttrsToList (n: v: lib.getOutput "profraw" v) coverageProfileDrvs;
|
|
|
|
mergedProfdata =
|
|
pkgs.runCommand "merged-profdata"
|
|
{
|
|
__structuredAttrs = true;
|
|
nativeBuildInputs = [ pkgs.llvmPackages.libllvm ];
|
|
inherit coverageProfiles;
|
|
}
|
|
''
|
|
rawProfiles=()
|
|
for dir in "''\${coverageProfiles[@]}"; do
|
|
rawProfiles+=($dir/*)
|
|
done
|
|
llvm-profdata merge -sparse -output $out "''\${rawProfiles[@]}"
|
|
'';
|
|
|
|
coverageReports =
|
|
let
|
|
nixComponentDrvs = lib.filter (lib.isDerivation) (lib.attrValues nixComponentsInstrumented);
|
|
in
|
|
pkgs.runCommand "code-coverage-report"
|
|
{
|
|
nativeBuildInputs = [
|
|
pkgs.llvmPackages.libllvm
|
|
pkgs.jq
|
|
];
|
|
__structuredAttrs = true;
|
|
nixComponents = nixComponentDrvs;
|
|
}
|
|
''
|
|
# ${toString (lib.map (v: v.src) nixComponentDrvs)}
|
|
|
|
binaryFiles=()
|
|
for dir in "''\${nixComponents[@]}"; do
|
|
readarray -t filesInDir < <(find "$dir" -type f -executable)
|
|
binaryFiles+=("''\${filesInDir[@]}")
|
|
done
|
|
|
|
arguments=$(concatStringsSep " -object " binaryFiles)
|
|
llvm-cov show $arguments -instr-profile ${mergedProfdata} -output-dir $out -format=html
|
|
|
|
{
|
|
echo "# Code coverage summary (generated via \`llvm-cov\`):"
|
|
echo
|
|
echo '```'
|
|
llvm-cov report $arguments -instr-profile ${mergedProfdata} -format=text -use-color=false
|
|
echo '```'
|
|
echo
|
|
} >> $out/index.txt
|
|
|
|
llvm-cov export $arguments -instr-profile ${mergedProfdata} -format=text > $out/coverage.json
|
|
|
|
mkdir -p $out/nix-support
|
|
|
|
coverageTotals=$(jq ".data[0].totals" $out/coverage.json)
|
|
|
|
# Mostly inline from pkgs/build-support/setup-hooks/make-coverage-analysis-report.sh [1],
|
|
# which we can't use here, because we rely on LLVM's infra for source code coverage collection.
|
|
# [1]: https://github.com/NixOS/nixpkgs/blob/67bb48c4c8e327417d6d5aa7e538244b209e852b/pkgs/build-support/setup-hooks/make-coverage-analysis-report.sh#L16
|
|
declare -A metricsArray=(["lineCoverage"]="lines" ["functionCoverage"]="functions" ["branchCoverage"]="branches")
|
|
|
|
for metricName in "''\${!metricsArray[@]}"; do
|
|
key="''\${metricsArray[$metricName]}"
|
|
metric=$(echo "$coverageTotals" | jq ".$key.percent * 10 | round / 10")
|
|
echo "$metricName $metric %" >> $out/nix-support/hydra-metrics
|
|
done
|
|
|
|
echo "report coverage $out" >> $out/nix-support/hydra-build-products
|
|
'';
|
|
in
|
|
assert withCoverage;
|
|
assert stdenv.cc.isClang;
|
|
{
|
|
inherit coverageProfileDrvs mergedProfdata coverageReports;
|
|
};
|
|
|
|
vmTests = {
|
|
}
|
|
# FIXME: when the curlS3 implementation is complete, it should also enable these tests.
|
|
// lib.optionalAttrs (withAWS == true) {
|
|
# S3 binary cache store test only runs when S3 support is enabled
|
|
inherit (nixosTests) s3-binary-cache-store;
|
|
}
|
|
// lib.optionalAttrs (!withSanitizers && !withCoverage) {
|
|
# evalNixpkgs uses non-instrumented components from hydraJobs, so only run it
|
|
# when not testing with sanitizers to avoid rebuilding nix
|
|
inherit (hydraJobs.tests) evalNixpkgs;
|
|
# FIXME: CI times out when building vm tests instrumented
|
|
inherit (nixosTests)
|
|
functional_user
|
|
githubFlakes
|
|
nix-docker
|
|
tarballFlakes
|
|
;
|
|
};
|
|
}
|