{ nixFlake ? builtins.getFlake ("git+file://" + toString ../../..), system ? builtins.currentSystem, pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system}, nixComponents ? ( nixFlake.lib.makeComponents { inherit pkgs; inherit getStdenv; } ), getStdenv ? p: p.stdenv, componentTestsPrefix ? "", withSanitizers ? false, withCoverage ? false, withAWS ? null, withCurlS3 ? null, ... }: let inherit (pkgs) lib; hydraJobs = nixFlake.hydraJobs; packages' = nixFlake.packages.${system}; stdenv = (getStdenv pkgs); enableSanitizersLayer = finalAttrs: prevAttrs: { mesonFlags = (prevAttrs.mesonFlags or [ ]) ++ [ (lib.mesonOption "b_sanitize" "address,undefined") ] ++ (lib.optionals stdenv.cc.isClang [ # https://www.github.com/mesonbuild/meson/issues/764 (lib.mesonBool "b_lundef" false) ]); }; collectCoverageLayer = finalAttrs: prevAttrs: { env = let # https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#the-code-coverage-workflow coverageFlags = [ "-fprofile-instr-generate" "-fcoverage-mapping" ]; in { CFLAGS = toString coverageFlags; CXXFLAGS = toString coverageFlags; }; # Done in a pre-configure hook, because $NIX_BUILD_TOP needs to be substituted. preConfigure = prevAttrs.preConfigure or "" + '' mappingFlag=" -fcoverage-prefix-map=$NIX_BUILD_TOP/${finalAttrs.src.name}=${finalAttrs.src}" CFLAGS+="$mappingFlag" CXXFLAGS+="$mappingFlag" ''; }; componentOverrides = (lib.optional withSanitizers enableSanitizersLayer) ++ (lib.optional withCoverage collectCoverageLayer); in rec { nixComponentsInstrumented = nixComponents.overrideScope ( final: prev: { nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; }; # Boehm is incompatible with ASAN. nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; # Override AWS configuration if specified nix-store = prev.nix-store.override ( lib.optionalAttrs (withAWS != null) { inherit withAWS; } // lib.optionalAttrs (withCurlS3 != null) { inherit withCurlS3; } ); mesonComponentOverrides = lib.composeManyExtensions componentOverrides; # Unclear how to make Perl bindings work with a dynamically linked ASAN. nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); # Import NixOS tests using the instrumented components nixosTests = import ../../../tests/nixos { inherit lib pkgs; nixComponents = nixComponentsInstrumented; nixpkgs = nixFlake.inputs.nixpkgs; inherit (nixFlake.inputs) nixpkgs-23-11; }; /** Top-level tests for the flake outputs, as they would be built by hydra. These tests generally can't be overridden to run with sanitizers. */ topLevel = { installerScriptForGHA = hydraJobs.installerScriptForGHA.${system}; installTests = hydraJobs.installTests.${system}; nixpkgsLibTests = hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${../../../doc/manual/rl-next} >$out ''; repl-completion = pkgs.callPackage ../../../tests/repl-completion.nix { inherit (packages') nix; }; /** Checks for our packaging expressions. This shouldn't build anything significant; just check that things (including derivations) are _set up_ correctly. */ packaging-overriding = let nix = packages'.nix; in assert (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src.patches == [ pkgs.emptyFile ]; if pkgs.stdenv.buildPlatform.isDarwin then lib.warn "packaging-overriding check currently disabled because of a permissions issue on macOS" pkgs.emptyFile else # If this fails, something might be wrong with how we've wired the scope, # or something could be broken in Nixpkgs. pkgs.testers.testEqualContents { assertion = "trivial patch does not change source contents"; expected = "${../../..}"; actual = # Same for all components; nix-util is an arbitrary pick (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src; }; }; componentTests = (lib.concatMapAttrs ( pkgName: pkg: lib.concatMapAttrs (testName: test: { "${componentTestsPrefix}${pkgName}-${testName}" = test; }) (pkg.tests or { }) ) nixComponentsInstrumented) // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) { "${componentTestsPrefix}nix-functional-tests" = nixComponentsInstrumented.nix-functional-tests; }; codeCoverage = let componentsTestsToProfile = (builtins.mapAttrs (n: v: nixComponentsInstrumented.${n}.tests.run) { "nix-util-tests" = { }; "nix-store-tests" = { }; "nix-fetchers-tests" = { }; "nix-expr-tests" = { }; "nix-flake-tests" = { }; }) // { inherit (nixComponentsInstrumented) nix-functional-tests; }; coverageProfileDrvs = lib.mapAttrs ( n: v: v.overrideAttrs ( finalAttrs: prevAttrs: { outputs = (prevAttrs.outputs or [ "out" ]) ++ [ "profraw" ]; env = { LLVM_PROFILE_FILE = "${placeholder "profraw"}/%m"; }; } ) ) componentsTestsToProfile; coverageProfiles = lib.mapAttrsToList (n: v: lib.getOutput "profraw" v) coverageProfileDrvs; mergedProfdata = pkgs.runCommand "merged-profdata" { __structuredAttrs = true; nativeBuildInputs = [ pkgs.llvmPackages.libllvm ]; inherit coverageProfiles; } '' rawProfiles=() for dir in "''\${coverageProfiles[@]}"; do rawProfiles+=($dir/*) done llvm-profdata merge -sparse -output $out "''\${rawProfiles[@]}" ''; coverageReports = let nixComponentDrvs = lib.filter (lib.isDerivation) (lib.attrValues nixComponentsInstrumented); in pkgs.runCommand "code-coverage-report" { nativeBuildInputs = [ pkgs.llvmPackages.libllvm pkgs.jq ]; __structuredAttrs = true; nixComponents = nixComponentDrvs; } '' # ${toString (lib.map (v: v.src) nixComponentDrvs)} binaryFiles=() for dir in "''\${nixComponents[@]}"; do readarray -t filesInDir < <(find "$dir" -type f -executable) binaryFiles+=("''\${filesInDir[@]}") done arguments=$(concatStringsSep " -object " binaryFiles) llvm-cov show $arguments -instr-profile ${mergedProfdata} -output-dir $out -format=html { echo "# Code coverage summary (generated via \`llvm-cov\`):" echo echo '```' llvm-cov report $arguments -instr-profile ${mergedProfdata} -format=text -use-color=false echo '```' echo } >> $out/index.txt llvm-cov export $arguments -instr-profile ${mergedProfdata} -format=text > $out/coverage.json mkdir -p $out/nix-support coverageTotals=$(jq ".data[0].totals" $out/coverage.json) # Mostly inline from pkgs/build-support/setup-hooks/make-coverage-analysis-report.sh [1], # which we can't use here, because we rely on LLVM's infra for source code coverage collection. # [1]: https://github.com/NixOS/nixpkgs/blob/67bb48c4c8e327417d6d5aa7e538244b209e852b/pkgs/build-support/setup-hooks/make-coverage-analysis-report.sh#L16 declare -A metricsArray=(["lineCoverage"]="lines" ["functionCoverage"]="functions" ["branchCoverage"]="branches") for metricName in "''\${!metricsArray[@]}"; do key="''\${metricsArray[$metricName]}" metric=$(echo "$coverageTotals" | jq ".$key.percent * 10 | round / 10") echo "$metricName $metric %" >> $out/nix-support/hydra-metrics done echo "report coverage $out" >> $out/nix-support/hydra-build-products ''; in assert withCoverage; assert stdenv.cc.isClang; { inherit coverageProfileDrvs mergedProfdata coverageReports; }; vmTests = { } # FIXME: when the curlS3 implementation is complete, it should also enable these tests. // lib.optionalAttrs (withAWS == true) { # S3 binary cache store test only runs when S3 support is enabled inherit (nixosTests) s3-binary-cache-store; } // lib.optionalAttrs (!withSanitizers && !withCoverage) { # evalNixpkgs uses non-instrumented components from hydraJobs, so only run it # when not testing with sanitizers to avoid rebuilding nix inherit (hydraJobs.tests) evalNixpkgs; # FIXME: CI times out when building vm tests instrumented inherit (nixosTests) functional_user githubFlakes nix-docker tarballFlakes ; }; }