From e2040aecacdf81dde9aad59a6bbc66162a04f668 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 6 Nov 2025 01:25:22 +0100 Subject: [PATCH 1/3] meson.build: Make schema checks optional --- meson.build | 7 ++++++- meson.options | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index f3158ea6d..c072a4821 100644 --- a/meson.build +++ b/meson.build @@ -60,4 +60,9 @@ if get_option('unit-tests') subproject('libflake-tests') endif subproject('nix-functional-tests') -subproject('json-schema-checks') +if get_option('json-schema-checks') + subproject('json-schema-checks') +endif +if get_option('kaitai-struct-checks') + subproject('kaitai-struct-checks') +endif diff --git a/meson.options b/meson.options index d2c9fa40c..2739b0c71 100644 --- a/meson.options +++ b/meson.options @@ -27,3 +27,17 @@ option( value : false, description : 'Build benchmarks (requires gbenchmark)', ) + +option( + 'kaitai-struct-checks', + type : 'boolean', + value : true, + description : 'Check the Kaitai Struct specifications (requires Kaitai Struct)', +) + +option( + 'json-schema-checks', + type : 'boolean', + value : true, + description : 'Check JSON schema validity of schemas and examples (requires jv)', +) From 1fa235b77c9cab1ab8dfdd97187ca2b6bf9dbf3d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 6 Nov 2025 01:29:25 +0100 Subject: [PATCH 2/3] devShells: Infer inputs from input closure boundary --- packaging/dev-shell.nix | 228 +++++++++++++++++++++++++++++++++++----- 1 file changed, 203 insertions(+), 25 deletions(-) diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 153e7a3eb..d50cab759 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -3,10 +3,118 @@ devFlake, }: +let + # Some helper functions + + /** + Compute a filtered closure of build inputs. + + Specifically, `buildInputsClosure cond startSet` computes the closure formed + by recursive application of `p: filter cond p.buildInputs ++ filter cond p.propagatedBuildInputs` + to `startSet`. + + Example: + ```nix + builtInputsClosure isInternal [ pkg1 pkg2 ] + => [ pkg1 pkg3 pkg2 pkg10 ] + ``` + + Note: order tbd + + Note: `startSet` is *NOT* filtered. + */ + buildInputsClosureCond = + cond: startSet: + let + closure = builtins.genericClosure { + startSet = map (d: { + key = d.drvPath; + value = d; + }) startSet; + operator = + d: + let + r = + map + (d': { + key = d'.drvPath; + value = d'; + }) + ( + lib.filter cond d.value.buildInputs or [ ] ++ lib.filter cond d.value.propagatedBuildInputs or [ ] + ); + in + r; + }; + in + map (item: item.value) closure; + + /** + `[ pkg1 pkg2 ]` -> `{ "...-pkg2.drv" = null; "...-pkg1.drv" = null }` + + Note: fairly arbitrary order (hash based). Use for efficient set membership test only. + */ + byDrvPath = + l: + lib.listToAttrs ( + map (c: { + name = + # Just a lookup key + builtins.unsafeDiscardStringContext c.drvPath; + value = null; + }) l + ); + + /** + Stable dedup. + + Unlike `listToAttrs` -> `attrValues`, this preserves the input ordering, + which is more predictable ("deterministic") than e.g. sorting store paths, + whose hashes affect the ordering on every change. + */ + # TODO: add to Nixpkgs lib, refer from uniqueStrings + dedupByString = + key: l: + let + r = + lib.foldl' + ( + a@{ list, set }: + elem: + let + k = builtins.unsafeDiscardStringContext (key elem); + in + if set ? ${k} then + a + else + let + # Note: O(n²) copying. Use linkedLists to concat them in one go at the end. + # https://github.com/NixOS/nixpkgs/pull/452088 + newList = [ elem ] ++ list; + newSet = set // { + ${k} = null; + }; + in + builtins.seq newList builtins.seq newSet { + list = newList; + set = newSet; + } + ) + { + list = [ ]; + set = { }; + } + l; + in + r.list; + +in + { pkgs }: +# TODO: don't use nix-util for this? pkgs.nixComponents2.nix-util.overrideAttrs ( - attrs: + finalAttrs: prevAttrs: let stdenv = pkgs.nixDependencies2.stdenv; @@ -21,13 +129,89 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( "-D${prefix}:${rest}"; havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + + activeComponents = buildInputsClosureCond isInternal ( + lib.attrValues (finalAttrs.passthru.config.getComponents allComponents) + ); + + allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2; + internalDrvs = byDrvPath ( + # Drop the attr names (not present in buildInputs anyway) + lib.attrValues allComponents + ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues allComponents) + ); + + isInternal = + dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; + in { - pname = "shell-for-" + attrs.pname; + pname = "shell-for-nix"; + + passthru = { + inherit activeComponents; + + # We use this attribute to store non-derivation values like functions and + # perhaps other things that are primarily for overriding and not the shell. + config = { + # Default getComponents + getComponents = + c: + builtins.removeAttrs c ( + lib.optionals (!havePerl) [ "nix-perl-bindings" ] + ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] + ); + }; + + /** + Produce a devShell for a given set of nix components + + Example: + + ```nix + shell.withActiveComponents (c: { + inherit (c) nix-util; + }) + ``` + */ + withActiveComponents = + f2: + finalAttrs.finalPackage.overrideAttrs ( + finalAttrs: prevAttrs: { + passthru = prevAttrs.passthru // { + config = prevAttrs.passthru.config // { + getComponents = f2; + }; + }; + } + ); + + small = + (finalAttrs.finalPackage.withActiveComponents (c: { + inherit (c) + nix-cli + nix-util-tests + nix-store-tests + nix-expr-tests + nix-fetchers-tests + nix-flake-tests + nix-functional-tests + # Currently required + nix-perl-bindings + ; + })).overrideAttrs + (o: { + mesonFlags = o.mesonFlags ++ [ + # TODO: infer from activeComponents or vice versa + "-Dkaitai-struct-checks=false" + "-Djson-schema-checks=false" + ]; + }); + }; # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop version = lib.fileContents ../.version; - name = attrs.pname; + name = finalAttrs.pname; installFlags = "sysconfdir=$(out)/etc"; shellHook = '' @@ -98,17 +282,9 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( nativeBuildInputs = let inputs = - attrs.nativeBuildInputs or [ ] - ++ pkgs.nixComponents2.nix-util.nativeBuildInputs - ++ pkgs.nixComponents2.nix-store.nativeBuildInputs - ++ pkgs.nixComponents2.nix-fetchers.nativeBuildInputs - ++ pkgs.nixComponents2.nix-expr.nativeBuildInputs - ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.nativeBuildInputs - ++ lib.optionals buildCanExecuteHost pkgs.nixComponents2.nix-manual.externalNativeBuildInputs - ++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs - ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs - ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs - ++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs + dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents) + ) ++ lib.optional ( !buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 @@ -117,9 +293,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) ) pkgs.buildPackages.mesonEmulatorHook ++ [ - pkgs.buildPackages.cmake pkgs.buildPackages.gnused - pkgs.buildPackages.changelog-d modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) pkgs.buildPackages.nixfmt-rfc-style @@ -136,18 +310,22 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( # from making its way into NIX_CFLAGS_COMPILE. lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; + propagatedNativeBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents + ) + ); + buildInputs = [ pkgs.gbenchmark ] - ++ attrs.buildInputs or [ ] - ++ pkgs.nixComponents2.nix-util.buildInputs - ++ pkgs.nixComponents2.nix-store.buildInputs - ++ pkgs.nixComponents2.nix-store-tests.externalBuildInputs - ++ pkgs.nixComponents2.nix-fetchers.buildInputs - ++ pkgs.nixComponents2.nix-expr.buildInputs - ++ pkgs.nixComponents2.nix-expr.externalPropagatedBuildInputs - ++ pkgs.nixComponents2.nix-cmd.buildInputs - ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.externalBuildInputs + ++ dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) + ) ++ lib.optional havePerl pkgs.perl; + + propagatedBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) + ); } ) From cb5b0c30aa5733b40cc70089fe3829cb1046c352 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 6 Nov 2025 01:52:38 +0100 Subject: [PATCH 3/3] Drop external*Inputs from packages Get rid of some manual package set resolution in favor of splicing again, too. Co-authored-by: John Ericson --- doc/manual/package.nix | 8 ++---- src/json-schema-checks/package.nix | 8 ++---- src/kaitai-struct-checks/package.nix | 24 ++++++---------- src/libexpr/package.nix | 5 ---- src/libstore-tests/package.nix | 12 +++----- src/perl/package.nix | 5 ---- tests/functional/package.nix | 43 ++++++++++++++++------------ 7 files changed, 41 insertions(+), 64 deletions(-) diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 81061b7a1..05368e34e 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -58,8 +58,8 @@ mkMesonDerivation (finalAttrs: { "man" ]; - # Hack for sake of the dev shell - passthru.externalNativeBuildInputs = [ + nativeBuildInputs = [ + nix-cli meson ninja (lib.getBin lowdown-unsandboxed) @@ -78,10 +78,6 @@ mkMesonDerivation (finalAttrs: { changelog-d ]; - nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [ - nix-cli - ]; - preConfigure = '' chmod u+w ./.version echo ${finalAttrs.version} > ./.version diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 5365fe75e..609d396be 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -34,15 +34,11 @@ mkMesonDerivation (finalAttrs: { outputs = [ "out" ]; - passthru.externalNativeBuildInputs = [ - jsonschema - ]; - nativeBuildInputs = [ meson ninja - ] - ++ finalAttrs.passthru.externalNativeBuildInputs; + jsonschema + ]; doCheck = true; diff --git a/src/kaitai-struct-checks/package.nix b/src/kaitai-struct-checks/package.nix index 97d56aabd..a466441d3 100644 --- a/src/kaitai-struct-checks/package.nix +++ b/src/kaitai-struct-checks/package.nix @@ -37,7 +37,15 @@ mkMesonDerivation (finalAttrs: { outputs = [ "out" ]; - passthru.externalNativeBuildInputs = [ + buildInputs = [ + gtest + kaitai-struct-cpp-stl-runtime + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config # This can go away when we bump up to 25.11 (kaitai-struct-compiler.overrideAttrs (finalAttrs: { version = "0.11"; @@ -48,20 +56,6 @@ mkMesonDerivation (finalAttrs: { })) ]; - passthru.externalBuildInputs = [ - gtest - kaitai-struct-cpp-stl-runtime - ]; - - buildInputs = finalAttrs.passthru.externalBuildInputs; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ] - ++ finalAttrs.passthru.externalNativeBuildInputs; - doCheck = true; mesonCheckFlags = [ "--print-errorlogs" ]; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index a67a8cc49..d0aef34e9 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -70,11 +70,6 @@ mkMesonLibrary (finalAttrs: { nix-util nix-store nix-fetchers - ] - ++ finalAttrs.passthru.externalPropagatedBuildInputs; - - # Hack for sake of the dev shell - passthru.externalPropagatedBuildInputs = [ boost nlohmann_json ] diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 90e6af519..ac547aca3 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -42,20 +42,16 @@ mkMesonExecutable (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - # Hack for sake of the dev shell - passthru.externalBuildInputs = [ + buildInputs = [ sqlite rapidcheck gtest - ] - ++ lib.optionals withBenchmarks [ - gbenchmark - ]; - - buildInputs = finalAttrs.passthru.externalBuildInputs ++ [ nix-store nix-store-c nix-store-test-support + ] + ++ lib.optionals withBenchmarks [ + gbenchmark ]; mesonFlags = [ diff --git a/src/perl/package.nix b/src/perl/package.nix index 10d84de77..96a41ae47 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -45,11 +45,6 @@ perl.pkgs.toPerlModule ( buildInputs = [ nix-store - ] - ++ finalAttrs.passthru.externalBuildInputs; - - # Hack for sake of the dev shell - passthru.externalBuildInputs = [ bzip2 libsodium ]; diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 6830a9e58..b3b314a50 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -2,7 +2,16 @@ lib, stdenv, mkMesonDerivation, - buildPackages, + + meson, + ninja, + pkg-config, + + jq, + git, + mercurial, + unixtools, + util-linux, nix-store, nix-expr, @@ -37,17 +46,20 @@ mkMesonDerivation ( ./. ]; - # Hack for sake of the dev shell. Need to "manually splice" since - # this isn't a specially-recognized list of dependencies. - passthru.externalNativeBuildInputs = [ - buildPackages.meson - buildPackages.ninja - buildPackages.pkg-config + nativeBuildInputs = [ + meson + ninja + pkg-config - buildPackages.jq - buildPackages.git - buildPackages.mercurial - buildPackages.unixtools.script + jq + git + mercurial + unixtools.script + + # Explicitly splice the hostHost variant to fix LLVM tests. The nix-cli + # has to be in PATH, but must come from the host context where it's built + # with libc++. + (nix-cli.__spliced.hostHost or nix-cli) ] ++ lib.optionals stdenv.hostPlatform.isLinux [ # For various sandboxing tests that needs a statically-linked shell, @@ -56,14 +68,7 @@ mkMesonDerivation ( # For Overlay FS tests need `mount`, `umount`, and `unshare`. # For `script` command (ensuring a TTY) # TODO use `unixtools` to be precise over which executables instead? - buildPackages.util-linux - ]; - - nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [ - # Explicitly splice the hostHost variant to fix LLVM tests. The nix-cli - # has to be in PATH, but must come from the host context where it's built - # with libc++. - (nix-cli.__spliced.hostHost or nix-cli) + util-linux ]; buildInputs = [