From 0d7b16da4dae9dc8f07756a6ef87caf17406e835 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Oct 2025 13:03:18 -0400 Subject: [PATCH 01/11] Split realisation protocol unit tests This will allow us to more accurately test dropping support for dependent realisations, by separating the tests that should not change from the tests that should. I do that change in PR #14247, but even if for some reasons we don't end up doing this soon, I think it is still good to separate the test data this way so we have the option of doing that at some point. --- src/libstore-tests/common-protocol.cc | 19 +++++++++++++-- .../common-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/common-protocol/realisation.bin | Bin 520 -> 384 bytes .../serve-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/serve-protocol/realisation.bin | Bin 520 -> 384 bytes .../worker-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/worker-protocol/realisation.bin | Bin 520 -> 384 bytes src/libstore-tests/serve-protocol.cc | 17 +++++++++++++ src/libstore-tests/worker-protocol.cc | 23 +++++++++++++++--- 9 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 src/libstore-tests/data/common-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-with-deps.bin diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 2c001957b..7c40e8cdb 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -114,13 +114,28 @@ CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +CHARACTERIZATION_TEST( + realisation_with_deps, + "realisation-with-deps", + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/data/common-protocol/realisation-with-deps.bin b/src/libstore-tests/data/common-protocol/realisation-with-deps.bin new file mode 100644 index 0000000000000000000000000000000000000000..54a78b64ebcf0726bd7da506434a316c7336e6e9 GIT binary patch literal 320 zcmXqFWB`L|rIgfy)V!3`ypo{Q#GK6H#FEVXykaG*YNg_gL?cr(E3-5UGs`r~)I=i- zBjco$L_;&vR0A^ubF;J*gOpSgV>7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -104,6 +113,14 @@ VERSIONED_CHARACTERIZATION_TEST( .outputName = "baz", }, }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index c4afde3bd..8f70e937b 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -150,13 +150,30 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -172,7 +189,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, }, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, From 144c66215b49bdadba7e11d2fd3ff9b108750274 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 18:01:37 -0400 Subject: [PATCH 02/11] JSON Schema for build trace entry Note, starting to make progress on #11895 by calling it this in the manual. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/build-trace-entry.md | 27 +++++++ doc/manual/source/protocols/json/meson.build | 1 + .../json/schema/build-trace-entry-v1 | 1 + .../json/schema/build-trace-entry-v1.yaml | 74 +++++++++++++++++++ doc/manual/source/store/build-trace.md | 2 +- src/json-schema-checks/build-trace-entry | 1 + src/json-schema-checks/meson.build | 9 +++ src/json-schema-checks/package.nix | 1 + 10 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/protocols/json/build-trace-entry.md create mode 120000 doc/manual/source/protocols/json/schema/build-trace-entry-v1 create mode 100644 doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml create mode 120000 src/json-schema-checks/build-trace-entry diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 7d29df3c3..e13c6f33d 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -37,6 +37,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path + ../../src/libstore-tests/data/realisation ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index b87bf93a3..580076ece 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -126,6 +126,7 @@ - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) + - [Build Trace Entry](protocols/json/build-trace-entry.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md new file mode 100644 index 000000000..8050a2840 --- /dev/null +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -0,0 +1,27 @@ +{{#include build-trace-entry-v1-fixed.md}} + +## Examples + +### Simple build trace entry + +```json +{{#include schema/build-trace-entry-v1/simple.json}} +``` + +### Build trace entry with dependencies + +```json +{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}} +``` + +### Build trace entry with signature + +```json +{{#include schema/build-trace-entry-v1/with-signature.json}} +``` + + \ No newline at end of file diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 7ebcff697..d8e94d68c 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -15,6 +15,7 @@ schemas = [ 'store-object-info-v1', 'derivation-v3', 'deriving-path-v1', + 'build-trace-entry-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1 b/doc/manual/source/protocols/json/schema/build-trace-entry-v1 new file mode 120000 index 000000000..0d02880a5 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/realisation \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml new file mode 100644 index 000000000..cabf2c350 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml @@ -0,0 +1,74 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json" +title: Build Trace Entry +description: | + A record of a successful build outcome for a specific derivation output. + + This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md) entry. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +type: object +required: + - id + - outPath + - dependentRealisations + - signatures +properties: + id: + type: string + title: Derivation Output ID + pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" + description: | + Unique identifier for the derivation output that was built. + + Format: `{hash-quotient-drv}!{output-name}` + + - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). + Begins with `sha256:`. + + - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") + + Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` + + outPath: + "$ref": "store-path-v1.yaml" + title: Output Store Path + description: | + The path to the store object that resulted from building this derivation for the given output name. + + dependentRealisations: + type: object + title: Underlying Base Build Trace + description: | + This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence. + + Keys are derivation output IDs (same format as the main `id` field). + Values are the store paths that those dependencies resolved to. + + As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries. + This is the set of base build trace entries that this derived build trace is derived from. + (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.) + + patternProperties: + "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": + $ref: "store-path-v1.yaml" + title: Dependent Store Path + description: Store path that this dependency resolved to during the build + additionalProperties: false + + signatures: + type: array + title: Build Signatures + description: | + A set of cryptographic signatures attesting to the authenticity of this build trace entry. + items: + type: string + title: Signature + description: A single cryptographic signature + +additionalProperties: false diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index 1086dcb88..8860bc6c7 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -29,7 +29,7 @@ And even in that case, a different result doesn't mean the original entry was a As such, the decision of whether to trust a counterparty's build trace is a fundamentally subject policy choice. Build trace entries are typically *signed* in order to enable arbitrary public-key-based trust polices. -## Derived build traces +## Derived build traces {#derived} Implementations that wish to memoize the above may also keep additional *derived* build trace entries that do map unresolved derivations. But if they do so, they *must* also keep the underlying base entries with resolved derivation keys around. diff --git a/src/json-schema-checks/build-trace-entry b/src/json-schema-checks/build-trace-entry new file mode 120000 index 000000000..9175e750e --- /dev/null +++ b/src/json-schema-checks/build-trace-entry @@ -0,0 +1 @@ +../../src/libstore-tests/data/realisation \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 67f553162..c2c7fbff4 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -54,6 +54,15 @@ schemas = [ 'single_built_built.json', ], }, + { + 'stem' : 'build-trace-entry', + 'schema' : schema_dir / 'build-trace-entry-v1.yaml', + 'files' : [ + 'simple.json', + 'with-dependent-realisations.json', + 'with-signature.json', + ], + }, ] # Derivation and Derivation output diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 160db003f..057a6e85b 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -23,6 +23,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path + ../../src/libstore-tests/data/realisation ../../src/libstore-tests/data/derivation ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info From 389bcba97a1295440a24c887840b1af3e73f0dd3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 13:42:52 -0400 Subject: [PATCH 03/11] JSON Impl and schema for `BuildResult` --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/json/build-result.md | 21 +++ doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/build-result-v1 | 1 + .../json/schema/build-result-v1.yaml | 136 +++++++++++++++++ src/json-schema-checks/build-result | 1 + src/json-schema-checks/meson.build | 9 ++ src/json-schema-checks/package.nix | 1 + src/libstore-tests/build-result.cc | 108 +++++++++++++ .../data/build-result/not-deterministic.json | 9 ++ .../data/build-result/output-rejected.json | 9 ++ .../data/build-result/success.json | 23 +++ src/libstore-tests/meson.build | 1 + src/libstore/build-result.cc | 142 ++++++++++++++++++ .../include/nix/store/build-result.hh | 3 + 16 files changed, 467 insertions(+) create mode 100644 doc/manual/source/protocols/json/build-result.md create mode 120000 doc/manual/source/protocols/json/schema/build-result-v1 create mode 100644 doc/manual/source/protocols/json/schema/build-result-v1.yaml create mode 120000 src/json-schema-checks/build-result create mode 100644 src/libstore-tests/build-result.cc create mode 100644 src/libstore-tests/data/build-result/not-deterministic.json create mode 100644 src/libstore-tests/data/build-result/output-rejected.json create mode 100644 src/libstore-tests/data/build-result/success.json diff --git a/doc/manual/package.nix b/doc/manual/package.nix index e13c6f33d..343e40016 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -41,6 +41,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 580076ece..5be3d6a90 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -127,6 +127,7 @@ - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) + - [Build Result](protocols/json/build-result.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/build-result.md b/doc/manual/source/protocols/json/build-result.md new file mode 100644 index 000000000..527e7bcc0 --- /dev/null +++ b/doc/manual/source/protocols/json/build-result.md @@ -0,0 +1,21 @@ +{{#include build-result-v1-fixed.md}} + +## Examples + +### Successful build + +```json +{{#include schema/build-result-v1/success.json}} +``` + +### Failed build (output rejected) + +```json +{{#include schema/build-result-v1/output-rejected.json}} +``` + +### Failed build (non-deterministic) + +```json +{{#include schema/build-result-v1/not-deterministic.json}} +``` \ No newline at end of file diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index d8e94d68c..c56de49c7 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -16,6 +16,7 @@ schemas = [ 'derivation-v3', 'deriving-path-v1', 'build-trace-entry-v1', + 'build-result-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-result-v1 b/doc/manual/source/protocols/json/schema/build-result-v1 new file mode 120000 index 000000000..a143d2c50 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml new file mode 100644 index 000000000..31f59a44d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -0,0 +1,136 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-result-v1.json" +title: Build Result +description: | + This schema describes the JSON representation of Nix's `BuildResult` type, which represents the result of building a derivation or substituting store paths. + + Build results can represent either successful builds (with built outputs) or various types of failures. + +oneOf: + - "$ref": "#/$defs/success" + - "$ref": "#/$defs/failure" +type: object +required: + - success + - status +properties: + timesBuilt: + type: integer + minimum: 0 + title: Times built + description: | + How many times this build was performed. + + startTime: + type: integer + minimum: 0 + title: Start time + description: | + The start time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + stopTime: + type: integer + minimum: 0 + title: Stop time + description: | + The stop time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + cpuUser: + type: integer + minimum: 0 + title: User CPU time + description: | + User CPU time the build took, in microseconds. + + cpuSystem: + type: integer + minimum: 0 + title: System CPU time + description: | + System CPU time the build took, in microseconds. + +"$defs": + success: + type: object + title: Successful Build Result + description: | + Represents a successful build with built outputs. + required: + - success + - status + - builtOutputs + properties: + success: + const: true + title: Success indicator + description: | + Always true for successful build results. + + status: + type: string + title: Success status + description: | + Status string for successful builds. + enum: + - "Built" + - "Substituted" + - "AlreadyValid" + - "ResolvesToAlreadyValid" + + builtOutputs: + type: object + title: Built outputs + description: | + A mapping from output names to their build trace entries. + additionalProperties: + "$ref": "build-trace-entry-v1.yaml" + + failure: + type: object + title: Failed Build Result + description: | + Represents a failed build with error information. + required: + - success + - status + - errorMsg + properties: + success: + const: false + title: Success indicator + description: | + Always false for failed build results. + + status: + type: string + title: Failure status + description: | + Status string for failed builds. + enum: + - "PermanentFailure" + - "InputRejected" + - "OutputRejected" + - "TransientFailure" + - "CachedFailure" + - "TimedOut" + - "MiscFailure" + - "DependencyFailed" + - "LogLimitExceeded" + - "NotDeterministic" + - "NoSubstituters" + - "HashMismatch" + + errorMsg: + type: string + title: Error message + description: | + Information about the error if the build failed. + + isNonDeterministic: + type: boolean + title: Non-deterministic flag + description: | + If timesBuilt > 1, whether some builds did not produce the same result. + + Note that 'isNonDeterministic = false' does not mean the build is deterministic, + just that we don't have evidence of non-determinism. diff --git a/src/json-schema-checks/build-result b/src/json-schema-checks/build-result new file mode 120000 index 000000000..8010d0fdd --- /dev/null +++ b/src/json-schema-checks/build-result @@ -0,0 +1 @@ +../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index c2c7fbff4..65a2651b7 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -150,6 +150,15 @@ schemas += [ 'impure.json', ], }, + { + 'stem' : 'build-result', + 'schema' : schema_dir / 'build-result-v1.yaml', + 'files' : [ + 'success.json', + 'output-rejected.json', + 'not-deterministic.json', + ], + }, # Match exact variant { 'stem' : 'store-object-info', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 057a6e85b..5365fe75e 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -28,6 +28,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result ./. ]; diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc new file mode 100644 index 000000000..85e799c2a --- /dev/null +++ b/src/libstore-tests/build-result.cc @@ -0,0 +1,108 @@ +#include + +#include "nix/store/build-result.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class BuildResultTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "build-result"; + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct BuildResultJsonTest : BuildResultTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(BuildResultJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(BuildResultJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +using namespace std::literals::chrono_literals; + +INSTANTIATE_TEST_SUITE_P( + BuildResultJSON, + BuildResultJsonTest, + ::testing::Values( + std::pair{ + "not-deterministic", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = false, // Note: This field is separate from the status + }}, + .timesBuilt = 1, + }, + }, + std::pair{ + "output-rejected", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + .isNonDeterministic = false, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + }, + std::pair{ + "success", + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs{ + { + "foo", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + }, + }, + { + "bar", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + }, + }, + }, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + })); + +} // namespace nix diff --git a/src/libstore-tests/data/build-result/not-deterministic.json b/src/libstore-tests/data/build-result/not-deterministic.json new file mode 100644 index 000000000..c24a15795 --- /dev/null +++ b/src/libstore-tests/data/build-result/not-deterministic.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "NotDeterministic", + "stopTime": 0, + "success": false, + "timesBuilt": 1 +} diff --git a/src/libstore-tests/data/build-result/output-rejected.json b/src/libstore-tests/data/build-result/output-rejected.json new file mode 100644 index 000000000..9494bf4ec --- /dev/null +++ b/src/libstore-tests/data/build-result/output-rejected.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 30, + "status": "OutputRejected", + "stopTime": 50, + "success": false, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json new file mode 100644 index 000000000..4baadb547 --- /dev/null +++ b/src/libstore-tests/data/build-result/success.json @@ -0,0 +1,23 @@ +{ + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 4d464ad89..f76df8bcb 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -54,6 +54,7 @@ deps_private += gtest subdir('nix-meson-build-support/common') sources = files( + 'build-result.cc', 'common-protocol.cc', 'content-address.cc', 'derivation-advanced-attrs.cc', diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index ecbd27b49..e3d9e9085 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -1,4 +1,6 @@ #include "nix/store/build-result.hh" +#include "nix/util/json-utils.hh" +#include namespace nix { @@ -11,4 +13,144 @@ std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Succes bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default; std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default; +static constexpr std::array, 4> successStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Success::e, #e} + ENUM_ENTRY(Built), + ENUM_ENTRY(Substituted), + ENUM_ENTRY(AlreadyValid), + ENUM_ENTRY(ResolvesToAlreadyValid), +#undef ENUM_ENTRY +}}; + +static std::string_view successStatusToString(BuildResult::Success::Status status) +{ + for (const auto & [enumVal, str] : successStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown success status: %d", static_cast(status)); +} + +static BuildResult::Success::Status successStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : successStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result success status '%s'", str); +} + +static constexpr std::array, 12> failureStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Failure::e, #e} + ENUM_ENTRY(PermanentFailure), + ENUM_ENTRY(InputRejected), + ENUM_ENTRY(OutputRejected), + ENUM_ENTRY(TransientFailure), + ENUM_ENTRY(CachedFailure), + ENUM_ENTRY(TimedOut), + ENUM_ENTRY(MiscFailure), + ENUM_ENTRY(DependencyFailed), + ENUM_ENTRY(LogLimitExceeded), + ENUM_ENTRY(NotDeterministic), + ENUM_ENTRY(NoSubstituters), + ENUM_ENTRY(HashMismatch), +#undef ENUM_ENTRY +}}; + +static std::string_view failureStatusToString(BuildResult::Failure::Status status) +{ + for (const auto & [enumVal, str] : failureStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown failure status: %d", static_cast(status)); +} + +static BuildResult::Failure::Status failureStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : failureStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result failure status '%s'", str); +} + } // namespace nix + +namespace nlohmann { + +using namespace nix; + +void adl_serializer::to_json(json & res, const BuildResult & br) +{ + res = json::object(); + + // Common fields + res["timesBuilt"] = br.timesBuilt; + res["startTime"] = br.startTime; + res["stopTime"] = br.stopTime; + + if (br.cpuUser.has_value()) { + res["cpuUser"] = br.cpuUser->count(); + } + if (br.cpuSystem.has_value()) { + res["cpuSystem"] = br.cpuSystem->count(); + } + + // Handle success or failure variant + std::visit( + overloaded{ + [&](const BuildResult::Success & success) { + res["success"] = true; + res["status"] = successStatusToString(success.status); + res["builtOutputs"] = success.builtOutputs; + }, + [&](const BuildResult::Failure & failure) { + res["success"] = false; + res["status"] = failureStatusToString(failure.status); + res["errorMsg"] = failure.errorMsg; + res["isNonDeterministic"] = failure.isNonDeterministic; + }, + }, + br.inner); +} + +BuildResult adl_serializer::from_json(const json & _json) +{ + auto & json = getObject(_json); + + BuildResult br; + + // Common fields + br.timesBuilt = getUnsigned(valueAt(json, "timesBuilt")); + br.startTime = getUnsigned(valueAt(json, "startTime")); + br.stopTime = getUnsigned(valueAt(json, "stopTime")); + + if (auto cpuUser = optionalValueAt(json, "cpuUser")) { + br.cpuUser = std::chrono::microseconds(getUnsigned(*cpuUser)); + } + if (auto cpuSystem = optionalValueAt(json, "cpuSystem")) { + br.cpuSystem = std::chrono::microseconds(getUnsigned(*cpuSystem)); + } + + // Determine success or failure based on success field + bool success = getBoolean(valueAt(json, "success")); + std::string statusStr = getString(valueAt(json, "status")); + + if (success) { + BuildResult::Success s; + s.status = successStatusFromString(statusStr); + s.builtOutputs = valueAt(json, "builtOutputs"); + br.inner = std::move(s); + } else { + BuildResult::Failure f; + f.status = failureStatusFromString(statusStr); + f.errorMsg = getString(valueAt(json, "errorMsg")); + f.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")); + br.inner = std::move(f); + } + + return br; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index 0446c4038..4739232f8 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -7,6 +7,7 @@ #include "nix/store/derived-path.hh" #include "nix/store/realisation.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -175,3 +176,5 @@ struct KeyedBuildResult : BuildResult }; } // namespace nix + +JSON_IMPL(nix::BuildResult) From 9daef9cca23c8d7f1658c3941eae59f1735c391e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Nov 2025 02:21:24 -0500 Subject: [PATCH 04/11] Clean up `DerivationOptions` unit tests We now test equality on whole strucks much more often, which avoids forgetting to test for specific fields. --- .../derivation-advanced-attrs.cc | 534 +++++++++--------- 1 file changed, 252 insertions(+), 282 deletions(-) diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 02bc8fa24..c994b26d5 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -32,6 +32,33 @@ public: * to worry about race conditions if the tests run concurrently. */ ExperimentalFeatureSettings mockXpSettings; + + /** + * Helper function to test getRequiredSystemFeatures for a given derivation file + */ + void testRequiredSystemFeatures(const std::string & fileName, const StringSet & expectedFeatures) + { + this->readTest(fileName, [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); + EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures); + }); + } + + /** + * Helper function to test DerivationOptions parsing and comparison + */ + void testDerivationOptions( + const std::string & fileName, const DerivationOptions & expected, const StringSet & expectedSystemFeatures) + { + this->readTest(fileName, [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); + + EXPECT_EQ(options, expected); + EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures); + }); + } }; class CaDerivationAdvancedAttrsTest : public DerivationAdvancedAttrsTest @@ -100,33 +127,35 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph); +static const DerivationOptions advancedAttributes_defaults = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) { this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, ""); - EXPECT_EQ(options.noChroot, false); - EXPECT_EQ(options.impureHostDeps, StringSet{}); - EXPECT_EQ(options.impureEnvVars, StringSet{}); - EXPECT_EQ(options.allowLocalNetworking, false); - EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); - { - auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); - ASSERT_TRUE(checksForAllOutputs_ != nullptr); - auto & checksForAllOutputs = *checksForAllOutputs_; + EXPECT_EQ(options, advancedAttributes_defaults); - EXPECT_EQ(checksForAllOutputs.allowedReferences, std::nullopt); - EXPECT_EQ(checksForAllOutputs.allowedRequisites, std::nullopt); - EXPECT_EQ(checksForAllOutputs.disallowedReferences, StringSet{}); - EXPECT_EQ(checksForAllOutputs.disallowedRequisites, StringSet{}); - } EXPECT_EQ(options.canBuildLocally(*this->store, got), false); EXPECT_EQ(options.willBuildLocally(*this->store, got), false); EXPECT_EQ(options.substitutesAllowed(), true); @@ -136,152 +165,124 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults) { - this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); - }); + testRequiredSystemFeatures("advanced-attributes-defaults.drv", {}); }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults) { - this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); - }); + testRequiredSystemFeatures("advanced-attributes-defaults.drv", {"ca-derivations"}); }; TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) { + DerivationOptions expected = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, + }; + this->readTest("advanced-attributes.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); - EXPECT_EQ(options.noChroot, true); - EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"}); - EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); - EXPECT_EQ(options.allowLocalNetworking, true); - EXPECT_EQ(options.canBuildLocally(*this->store, got), false); - EXPECT_EQ(options.willBuildLocally(*this->store, got), false); + // Reset fields that vary between test cases to enable whole-object comparison + options.outputChecks = DerivationOptions::OutputChecks{.ignoreSelfRefs = true}; + options.exportReferencesGraph = {}; + + EXPECT_EQ(options, expected); + EXPECT_EQ(options.substitutesAllowed(), false); EXPECT_EQ(options.useUidRange(got), true); }); }; -TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) +DerivationOptions advancedAttributes_ia = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + .disallowedReferences = StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + .allowedRequisites = StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, + .disallowedRequisites = StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}}, + {"refs2", {"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_ia) { - this->readTest("advanced-attributes.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"}); +}; - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", - }, - }, - { - "refs2", - { - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", - }, - }, - })); - - { - auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); - ASSERT_TRUE(checksForAllOutputs_ != nullptr); - auto & checksForAllOutputs = *checksForAllOutputs_; - - EXPECT_EQ( - checksForAllOutputs.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); - EXPECT_EQ( - checksForAllOutputs.allowedRequisites, - StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); - EXPECT_EQ( - checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); - EXPECT_EQ( - checksForAllOutputs.disallowedRequisites, - StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); +DerivationOptions advancedAttributes_ca = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}, + .disallowedReferences = StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}, + .allowedRequisites = StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}, + .disallowedRequisites = StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}}, + {"refs2", {"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) { - this->readTest("advanced-attributes.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"}); +}; - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", - }, - }, - { - "refs2", - { - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", - }, - }, - })); - - { - auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); - ASSERT_TRUE(checksForAllOutputs_ != nullptr); - auto & checksForAllOutputs = *checksForAllOutputs_; - - EXPECT_EQ( - checksForAllOutputs.allowedReferences, - StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); - EXPECT_EQ( - checksForAllOutputs.allowedRequisites, - StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); - EXPECT_EQ( - checksForAllOutputs.disallowedReferences, - StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); - EXPECT_EQ( - checksForAllOutputs.disallowedRequisites, - StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - systemFeatures.insert("ca-derivations"); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); +DerivationOptions advancedAttributes_structuredAttrs_defaults = { + .outputChecks = std::map{}, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, }; TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_defaults) @@ -289,26 +290,11 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, ""); - EXPECT_EQ(options.noChroot, false); - EXPECT_EQ(options.impureHostDeps, StringSet{}); - EXPECT_EQ(options.impureEnvVars, StringSet{}); - EXPECT_EQ(options.allowLocalNetworking, false); - EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{}); - - { - auto * checksPerOutput_ = std::get_if<1>(&options.outputChecks); - ASSERT_TRUE(checksPerOutput_ != nullptr); - auto & checksPerOutput = *checksPerOutput_; - - EXPECT_EQ(checksPerOutput.size(), 0u); - } + EXPECT_EQ(options, advancedAttributes_structuredAttrs_defaults); EXPECT_EQ(options.canBuildLocally(*this->store, got), false); EXPECT_EQ(options.willBuildLocally(*this->store, got), false); @@ -319,55 +305,61 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) { - this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); - }); + testRequiredSystemFeatures("advanced-attributes-structured-attrs-defaults.drv", {}); }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) { - this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); - }); + testRequiredSystemFeatures("advanced-attributes-structured-attrs-defaults.drv", {"ca-derivations"}); }; TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { + DerivationOptions expected = { + .outputChecks = + std::map{ + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, + }; + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs); - EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); - EXPECT_EQ(options.noChroot, true); - EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"}); - EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); - EXPECT_EQ(options.allowLocalNetworking, true); - + // Reset fields that vary between test cases to enable whole-object comparison { - auto output_ = get(std::get<1>(options.outputChecks), "dev"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.maxSize, 789); - EXPECT_EQ(output.maxClosureSize, 5909); + // Delete all keys but "dev" in options.outputChecks + auto * outputChecksMapP = + std::get_if>(&options.outputChecks); + ASSERT_TRUE(outputChecksMapP); + auto & outputChecksMap = *outputChecksMapP; + auto devEntry = outputChecksMap.find("dev"); + ASSERT_TRUE(devEntry != outputChecksMap.end()); + auto devChecks = devEntry->second; + outputChecksMap.clear(); + outputChecksMap.emplace("dev", std::move(devChecks)); } + options.exportReferencesGraph = {}; + + EXPECT_EQ(options, expected); EXPECT_EQ(options.canBuildLocally(*this->store, got), false); EXPECT_EQ(options.willBuildLocally(*this->store, got), false); @@ -376,112 +368,90 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) }); }; +DerivationOptions advancedAttributes_structuredAttrs_ia = { + .outputChecks = + std::map{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + .allowedRequisites = StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + .disallowedRequisites = StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = + { + {"refs1", {"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}}, + {"refs2", {"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) { - this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", - }, - }, - { - "refs2", - { - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv", - }, - }, - })); + testDerivationOptions( + "advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"}); +}; +DerivationOptions advancedAttributes_structuredAttrs_ca = { + .outputChecks = + std::map{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}, + .allowedRequisites = StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}, + .disallowedRequisites = StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = { - { - auto output_ = get(std::get<1>(options.outputChecks), "out"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); - EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); - } - - { - auto output_ = get(std::get<1>(options.outputChecks), "bin"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); - EXPECT_EQ( - output.disallowedRequisites, StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); - } - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); + {"refs1", {"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}}, + {"refs2", {"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, }; TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) { - this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { - auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - - auto drvPath = writeDerivation(*this->store, got, NoRepair, true); - - DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs); - - EXPECT_EQ( - options.exportReferencesGraph, - (ExportReferencesMap{ - { - "refs1", - { - "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", - }, - }, - { - "refs2", - { - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv", - }, - }, - })); - - { - { - auto output_ = get(std::get<1>(options.outputChecks), "out"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.allowedReferences, StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); - EXPECT_EQ(output.allowedRequisites, StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); - } - - { - auto output_ = get(std::get<1>(options.outputChecks), "bin"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ( - output.disallowedReferences, StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); - EXPECT_EQ( - output.disallowedRequisites, StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); - } - } - - StringSet systemFeatures{"rainbow", "uid-range"}; - systemFeatures.insert("ca-derivations"); - - EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - }); + testDerivationOptions( + "advanced-attributes-structured-attrs.drv", + advancedAttributes_structuredAttrs_ca, + {"rainbow", "uid-range", "ca-derivations"}); }; } // namespace nix From d05e85e5befbc7fcc19a28d459ed17f5e98c6f23 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Nov 2025 01:22:20 -0500 Subject: [PATCH 05/11] Fix `DerivationOptions` JSON implementation and test --- .../data/derivation/ca/all_set.json | 46 +++++++++++++ .../ca/structuredAttrs_all_set.json | 66 +++++++++++++++++++ .../data/derivation/ia/all_set.json | 46 +++++++++++++ .../data/derivation/ia/defaults.json | 24 +++++++ .../ia/structuredAttrs_all_set.json | 66 +++++++++++++++++++ .../ia/structuredAttrs_defaults.json | 16 +++++ .../derivation-advanced-attrs.cc | 25 ++++++- src/libstore/derivation-options.cc | 33 ++++++++-- src/libstore/derivations.cc | 2 +- src/libutil/include/nix/util/json-utils.hh | 11 ++++ src/libutil/json-utils.cc | 9 +-- 11 files changed, 326 insertions(+), 18 deletions(-) create mode 100644 src/libstore-tests/data/derivation/ca/all_set.json create mode 100644 src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json create mode 100644 src/libstore-tests/data/derivation/ia/all_set.json create mode 100644 src/libstore-tests/data/derivation/ia/defaults.json create mode 100644 src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json create mode 100644 src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json diff --git a/src/libstore-tests/data/derivation/ca/all_set.json b/src/libstore-tests/data/derivation/ca/all_set.json new file mode 100644 index 000000000..e06eada01 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/all_set.json @@ -0,0 +1,46 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "refs2": [ + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "allowedRequisites": [ + "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z" + ], + "disallowedReferences": [ + "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g" + ], + "disallowedRequisites": [ + "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8" + ], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json b/src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json new file mode 100644 index 000000000..2a321897c --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json @@ -0,0 +1,66 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "refs2": [ + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "perOutput": { + "bin": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [ + "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g" + ], + "disallowedRequisites": [ + "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8" + ], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + }, + "dev": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": 5909, + "maxSize": 789 + }, + "out": { + "allowedReferences": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "allowedRequisites": [ + "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z" + ], + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + } + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/all_set.json b/src/libstore-tests/data/derivation/ia/all_set.json new file mode 100644 index 000000000..62b6cdf97 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/all_set.json @@ -0,0 +1,46 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "refs2": [ + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "allowedRequisites": [ + "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev" + ], + "disallowedReferences": [ + "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar" + ], + "disallowedRequisites": [ + "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" + ], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/defaults.json b/src/libstore-tests/data/derivation/ia/defaults.json new file mode 100644 index 000000000..79a68989c --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/defaults.json @@ -0,0 +1,24 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": {}, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json b/src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json new file mode 100644 index 000000000..0fa383589 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json @@ -0,0 +1,66 @@ +{ + "additionalSandboxProfile": "sandcastle", + "allowLocalNetworking": true, + "allowSubstitutes": false, + "exportReferencesGraph": { + "refs1": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "refs2": [ + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "impureHostDeps": [ + "/usr/bin/ditto" + ], + "noChroot": true, + "outputChecks": { + "perOutput": { + "bin": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [ + "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar" + ], + "disallowedRequisites": [ + "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" + ], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + }, + "dev": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": 5909, + "maxSize": 789 + }, + "out": { + "allowedReferences": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "allowedRequisites": [ + "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev" + ], + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": false, + "maxClosureSize": null, + "maxSize": null + } + } + }, + "passAsFile": [], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json b/src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json new file mode 100644 index 000000000..d898d85f5 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/structuredAttrs_defaults.json @@ -0,0 +1,16 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": {}, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index c994b26d5..41538cdcc 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -10,13 +10,15 @@ #include "nix/util/json-utils.hh" #include "nix/store/tests/libstore.hh" -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { using namespace nlohmann; -class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest +class DerivationAdvancedAttrsTest : public JsonCharacterizationTest, + public JsonCharacterizationTest, + public LibStoreTest { protected: std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "ia"; @@ -454,4 +456,23 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) {"rainbow", "uid-range", "ca-derivations"}); }; +#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ + TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ + { \ + this->JsonCharacterizationTest::readJsonTest(#VAR, advancedAttributes_##VAR2); \ + } \ + TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ + { \ + this->JsonCharacterizationTest::writeJsonTest(#VAR, advancedAttributes_##VAR2); \ + } + +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, all_set, ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, all_set, ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_defaults, structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ca) + +#undef TEST_JSON_OPTIONS + } // namespace nix diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index bd9704b44..75313841c 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -176,13 +176,26 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt return {}; }; - checks.allowedReferences = get_("allowedReferences"); - checks.allowedRequisites = get_("allowedRequisites"); - checks.disallowedReferences = get_("disallowedReferences").value_or(StringSet{}); - checks.disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}); - ; - - res.insert_or_assign(outputName, std::move(checks)); + res.insert_or_assign( + outputName, + OutputChecks{ + .maxSize = [&]() -> std::optional { + if (auto maxSize = get(output, "maxSize")) + return maxSize->get(); + else + return std::nullopt; + }(), + .maxClosureSize = [&]() -> std::optional { + if (auto maxClosureSize = get(output, "maxClosureSize")) + return maxClosureSize->get(); + else + return std::nullopt; + }(), + .allowedReferences = get_("allowedReferences"), + .disallowedReferences = get_("disallowedReferences").value_or(StringSet{}), + .allowedRequisites = get_("allowedRequisites"), + .disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}), + }); } } return res; @@ -364,6 +377,7 @@ DerivationOptions adl_serializer::from_json(const json & json .unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"), .passAsFile = getStringSet(valueAt(json, "passAsFile")), + .exportReferencesGraph = getMap(getObject(valueAt(json, "exportReferencesGraph")), getStringSet), .additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")), .noChroot = getBoolean(valueAt(json, "noChroot")), @@ -396,6 +410,7 @@ void adl_serializer::to_json(json & json, const DerivationOpt json["unsafeDiscardReferences"] = o.unsafeDiscardReferences; json["passAsFile"] = o.passAsFile; + json["exportReferencesGraph"] = o.exportReferencesGraph; json["additionalSandboxProfile"] = o.additionalSandboxProfile; json["noChroot"] = o.noChroot; @@ -423,6 +438,8 @@ DerivationOptions::OutputChecks adl_serializer: return { .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), + .maxSize = ptrToOwned(getNullable(valueAt(json, "maxSize"))), + .maxClosureSize = ptrToOwned(getNullable(valueAt(json, "maxClosureSize"))), .allowedReferences = ptrToOwned(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")), .allowedRequisites = ptrToOwned(getNullable(valueAt(json, "allowedRequisites"))), @@ -433,6 +450,8 @@ DerivationOptions::OutputChecks adl_serializer: void adl_serializer::to_json(json & json, const DerivationOptions::OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; + json["maxSize"] = c.maxSize; + json["maxClosureSize"] = c.maxClosureSize; json["allowedReferences"] = c.allowedReferences; json["disallowedReferences"] = c.disallowedReferences; json["allowedRequisites"] = c.allowedRequisites; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 726143db2..e6ac08fd9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1404,7 +1404,7 @@ void adl_serializer::to_json(json & res, const Derivation & d) { auto & inputsList = res["inputSrcs"]; - inputsList = nlohmann::json ::array(); + inputsList = nlohmann::json::array(); for (auto & input : d.inputSrcs) inputsList.emplace_back(input); } diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index 51ebb2b6c..7a3fe4f36 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -59,6 +59,17 @@ auto getInteger(const nlohmann::json & value) -> std::enable_if_t +std::map getMap(const nlohmann::json::object_t & jsonObject, auto && f) +{ + std::map map; + + for (const auto & [key, value] : jsonObject) + map.insert_or_assign(key, f(value)); + + return map; +} + const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value); Strings getStringList(const nlohmann::json & value); StringMap getStringMap(const nlohmann::json & value); diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 1502384e9..80779541e 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -91,14 +91,7 @@ Strings getStringList(const nlohmann::json & value) StringMap getStringMap(const nlohmann::json & value) { - auto & jsonObject = getObject(value); - - StringMap stringMap; - - for (const auto & [key, value] : jsonObject) - stringMap[getString(key)] = getString(value); - - return stringMap; + return getMap>(getObject(value), getString); } StringSet getStringSet(const nlohmann::json & value) From b2f0472fe256b074a7aa7320bc54437886651a2b Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Tue, 4 Nov 2025 21:20:19 +0100 Subject: [PATCH 06/11] parser.y: allocate `Expr`s in the allocator --- src/libexpr/include/nix/expr/nixexpr.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 1db79c58b..08d39cd87 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -813,7 +813,7 @@ public: [[gnu::always_inline]] C * add(auto &&... args) { - return new C(std::forward(args)...); + return alloc.new_object(std::forward(args)...); } // we define some calls to add explicitly so that the argument can be passed in as initializer lists @@ -822,7 +822,7 @@ public: C * add(const PosIdx & pos, Expr * fun, std::vector && args) requires(std::same_as) { - return new C(pos, fun, std::move(args)); + return alloc.new_object(pos, fun, std::move(args)); } template @@ -830,7 +830,7 @@ public: C * add(const PosIdx & pos, Expr * fun, std::vector && args, PosIdx && cursedOrEndPos) requires(std::same_as) { - return new C(pos, fun, std::move(args), std::move(cursedOrEndPos)); + return alloc.new_object(pos, fun, std::move(args), std::move(cursedOrEndPos)); } template From e7b274f85a1f75e3c4dba445f081dd7862586d07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Nov 2025 23:38:45 -0500 Subject: [PATCH 07/11] `DerivationBuilderParams` have reference to `BasicDerivation` Have one to that instead of one to `Derivation`. `DerivationBuilder` doesn't need `inputDrvs`, so `BasicDerivation` suffices. (In fact, it doesn't need `inputSrcs` either, but we don't yet hve a type to exclude that.) --- src/libstore/include/nix/store/build/derivation-builder.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index 5fad26e83..5eed38462 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -62,7 +62,7 @@ struct DerivationBuilderParams /** * The derivation stored at drvPath. */ - const Derivation & drv; + const BasicDerivation & drv; /** * The derivation options of `drv`. From c1317017e902f34631f2f4598710114d36e84ee4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Nov 2025 13:06:08 +0100 Subject: [PATCH 08/11] Don't crash on flakerefs containing newlines Fixes #14311. --- src/libflake/flakeref.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index a26f269c3..ed6b657ac 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -109,7 +109,8 @@ std::pair parsePathFlakeRefWithFragment( std::smatch match; auto succeeds = std::regex_match(url, match, pathFlakeRegex); - assert(succeeds); + if (!succeeds) + throw Error("invalid flakeref '%s'", url); auto path = match[1].str(); auto query = decodeQuery(match[3].str(), /*lenient=*/true); auto fragment = percentDecode(match[5].str()); From 0c37a62207f52aa9d58b8d18a4860dec3a270f72 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 00:24:12 -0400 Subject: [PATCH 09/11] Change JSON derivation format in two ways - Use canonical content address JSON format for floating content addressed derivation outputs This keeps it more consistent. - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) This will allow for an easier to use, but less compact, alternative where `srcs` is just a list of derived paths. It also allows for other experiments for derivations with a different input structure, as I suspect will be needed for secure build traces. --- .../source/protocols/json/derivation.md | 4 +- doc/manual/source/protocols/json/meson.build | 2 +- ...{derivation-v3.yaml => derivation-v4.yaml} | 114 ++++++++++-------- src/json-schema-checks/meson.build | 14 +-- .../ca/advanced-attributes-defaults.json | 8 +- ...-attributes-structured-attrs-defaults.json | 8 +- .../advanced-attributes-structured-attrs.json | 38 +++--- .../derivation/ca/advanced-attributes.json | 38 +++--- .../data/derivation/ca/self-contained.json | 8 +- .../data/derivation/dyn-dep-derivation.json | 50 ++++---- .../ia/advanced-attributes-defaults.json | 8 +- ...-attributes-structured-attrs-defaults.json | 8 +- .../advanced-attributes-structured-attrs.json | 38 +++--- .../derivation/ia/advanced-attributes.json | 38 +++--- .../data/derivation/output-caFixedFlat.json | 7 +- .../data/derivation/output-caFixedNAR.json | 7 +- .../data/derivation/output-caFixedText.json | 7 +- .../data/derivation/simple-derivation.json | 26 ++-- src/libstore-tests/nix_api_store.cc | 8 +- src/libstore/derivations.cc | 108 +++++++++-------- tests/functional/dyn-drv/non-trivial.nix | 10 +- 21 files changed, 298 insertions(+), 251 deletions(-) rename doc/manual/source/protocols/json/schema/{derivation-v3.yaml => derivation-v4.yaml} (78%) diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index a4a4ea79d..6eafb255e 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,7 +1,7 @@ -{{#include derivation-v3-fixed.md}} +{{#include derivation-v4-fixed.md}} diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c56de49c7..c0b8416d7 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -13,7 +13,7 @@ schemas = [ 'content-address-v1', 'store-path-v1', 'store-object-info-v1', - 'derivation-v3', + 'derivation-v4', 'deriving-path-v1', 'build-trace-entry-v1', 'build-result-v1', diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml similarity index 78% rename from doc/manual/source/protocols/json/schema/derivation-v3.yaml rename to doc/manual/source/protocols/json/schema/derivation-v4.yaml index fa68adcb1..2528f7502 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -1,8 +1,8 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v4.json" title: Derivation description: | - Experimental JSON representation of a Nix derivation (version 3). + Experimental JSON representation of a Nix derivation (version 4). This schema describes the JSON representation of Nix's `Derivation` type. @@ -17,8 +17,7 @@ required: - name - version - outputs - - inputSrcs - - inputDrvs + - inputs - system - builder - args @@ -32,10 +31,10 @@ properties: Used when calculating store paths for the derivation’s outputs. version: - const: 3 - title: Format version (must be 3) + const: 4 + title: Format version (must be 4) description: | - Must be `3`. + Must be `4`. This is a guard that allows us to continue evolving this format. The choice of `3` is fairly arbitrary, but corresponds to this informal version: @@ -47,6 +46,12 @@ properties: - Version 3: Drop store dir from store paths, just include base name. + - Version 4: Two cleanups, batched together to lesson churn: + + - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) + + - Use canonical content address JSON format for floating content addressed derivation outputs. + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. outputs: @@ -70,47 +75,56 @@ properties: additionalProperties: "$ref": "#/$defs/output/overall" - inputSrcs: - type: array - title: Input source paths - description: | - List of store paths on which this derivation depends. - - > **Example** - > - > ```json - > "inputSrcs": [ - > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", - > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" - > ] - > ``` - items: - $ref: "store-path-v1.yaml" - - inputDrvs: + inputs: type: object - title: Input derivations + title: Derivation inputs description: | - Mapping of derivation paths to lists of output names they provide. - - > **Example** - > - > ```json - > "inputDrvs": { - > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] - > } - > ``` - > - > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. - patternProperties: - "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": - title: Store Path + Input dependencies for the derivation, organized into source paths and derivation dependencies. + required: + - srcs + - drvs + properties: + srcs: + type: array + title: Input source paths description: | - A store path to a derivation, mapped to the outputs of that derivation. - oneOf: - - "$ref": "#/$defs/outputNames" - - "$ref": "#/$defs/dynamicOutputs" + List of store paths on which this derivation depends. + + > **Example** + > + > ```json + > "srcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + items: + $ref: "store-path-v1.yaml" + drvs: + type: object + title: Input derivations + description: | + Mapping of derivation paths to lists of output names they provide. + + > **Example** + > + > ```json + > "drvs": { + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > } + > ``` + > + > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + patternProperties: + "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": + title: Store Path + description: | + A store path to a derivation, mapped to the outputs of that derivation. + oneOf: + - "$ref": "#/$defs/outputNames" + - "$ref": "#/$defs/dynamicOutputs" + additionalProperties: false additionalProperties: false system: @@ -189,24 +203,18 @@ properties: The output is content-addressed, and the content-address is fixed in advance. See [Fixed-output content-addressing](@docroot@/store/derivation/outputs/content-address.md#fixed) for more details. - type: object + "$ref": "./content-address-v1.yaml" required: - method - - hashAlgo - hash properties: method: - "$ref": "./content-address-v1.yaml#/$defs/method" description: | Method of content addressing used for this output. - hashAlgo: - title: Hash algorithm - "$ref": "./hash-v1.yaml#/$defs/algorithm" hash: - type: string title: Expected hash value description: | - The expected content hash in base-16. + The expected content hash. additionalProperties: false caFloating: diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 65a2651b7..fedacedeb 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -70,7 +70,7 @@ schemas += [ # Match overall { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml', + 'schema' : schema_dir / 'derivation-v4.yaml', 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', @@ -78,7 +78,7 @@ schemas += [ }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/overall', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/overall', 'files' : [ 'output-caFixedFlat.json', 'output-caFixedNAR.json', @@ -92,14 +92,14 @@ schemas += [ # Match exact variant { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/inputAddressed', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/inputAddressed', 'files' : [ 'output-inputAddressed.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFixed', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFixed', 'files' : [ 'output-caFixedFlat.json', 'output-caFixedNAR.json', @@ -108,21 +108,21 @@ schemas += [ }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFloating', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFloating', 'files' : [ 'output-caFloating.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/deferred', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/deferred', 'files' : [ 'output-deferred.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/impure', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/impure', 'files' : [ 'output-impure.json', ], diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json index eb4bd4f3d..781b4cb14 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -12,8 +12,10 @@ "outputHashMode": "recursive", "system": "my-system" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-defaults", "outputs": { "out": { @@ -22,5 +24,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 3a4a3079b..7437b51ef 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -8,8 +8,10 @@ "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { @@ -33,5 +35,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index b10355af7..2a4e70558 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -9,25 +9,27 @@ "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, - "inputDrvs": { - "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] }, - "inputSrcs": [ - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" - ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { @@ -101,5 +103,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index d66882036..55dbe62e0 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -25,25 +25,27 @@ "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" }, - "inputDrvs": { - "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] }, - "inputSrcs": [ - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" - ], "name": "advanced-attributes", "outputs": { "out": { @@ -52,5 +54,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/self-contained.json b/src/libstore-tests/data/derivation/ca/self-contained.json index 331beb7be..c05710140 100644 --- a/src/libstore-tests/data/derivation/ca/self-contained.json +++ b/src/libstore-tests/data/derivation/ca/self-contained.json @@ -10,8 +10,10 @@ "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", "system": "x86_64-linux" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "myname", "outputs": { "out": { @@ -20,5 +22,5 @@ } }, "system": "x86_64-linux", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/dyn-dep-derivation.json b/src/libstore-tests/data/derivation/dyn-dep-derivation.json index 1a9f54c53..1793c5f2d 100644 --- a/src/libstore-tests/data/derivation/dyn-dep-derivation.json +++ b/src/libstore-tests/data/derivation/dyn-dep-derivation.json @@ -7,33 +7,35 @@ "env": { "BIG_BAD": "WOLF" }, - "inputDrvs": { - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": [ - "kitten" - ] + "inputs": { + "drvs": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } }, - "goose": { - "dynamicOutputs": {}, - "outputs": [ - "gosling" - ] - } - }, - "outputs": [ - "cat", - "dog" - ] - } + "outputs": [ + "cat", + "dog" + ] + } + }, + "srcs": [ + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ] }, - "inputSrcs": [ - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], "name": "dyn-dep-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json index 0fa543f21..898762123 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json @@ -10,8 +10,10 @@ "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", "system": "my-system" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-defaults", "outputs": { "out": { @@ -19,5 +21,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index e02392ea1..c51095986 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -8,8 +8,10 @@ "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { @@ -29,5 +31,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index 9230b06b6..e07d1294b 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -9,25 +9,27 @@ "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev", "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, - "inputDrvs": { - "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] }, - "inputSrcs": [ - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" - ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { @@ -96,5 +98,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index ba5911c91..372b4fbb9 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -23,25 +23,27 @@ "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" }, - "inputDrvs": { - "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] }, - "inputSrcs": [ - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" - ], "name": "advanced-attributes", "outputs": { "out": { @@ -49,5 +51,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json index e6a0123f6..9a38608b3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedFlat.json +++ b/src/libstore-tests/data/derivation/output-caFixedFlat.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "flat" } diff --git a/src/libstore-tests/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json index b57e065a9..767c605a3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedNAR.json +++ b/src/libstore-tests/data/derivation/output-caFixedNAR.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "nar" } diff --git a/src/libstore-tests/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json index 84778509e..a04f1ff2a 100644 --- a/src/libstore-tests/data/derivation/output-caFixedText.json +++ b/src/libstore-tests/data/derivation/output-caFixedText.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "text" } diff --git a/src/libstore-tests/data/derivation/simple-derivation.json b/src/libstore-tests/data/derivation/simple-derivation.json index 41a049aef..04129a096 100644 --- a/src/libstore-tests/data/derivation/simple-derivation.json +++ b/src/libstore-tests/data/derivation/simple-derivation.json @@ -7,20 +7,22 @@ "env": { "BIG_BAD": "WOLF" }, - "inputDrvs": { - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } + "inputs": { + "drvs": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "srcs": [ + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ] }, - "inputSrcs": [ - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], "name": "simple-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 228b8069f..bf411053a 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -636,7 +636,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) auto outj_ph = nix::hashPlaceholder("outj"); std::string drvJson = R"({ - "version": 3, + "version": 4, "name": "multi-output-test", "system": ")" + nix::settings.thisSystem.get() + R"(", @@ -668,8 +668,10 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) "outa": ")" + outa_ph + R"(" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "outputs": { "outd": { "hashAlgo": "sha256", "method": "nar" }, "outf": { "hashAlgo": "sha256", "method": "nar" }, diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index e6ac08fd9..31ca167f9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1293,15 +1293,13 @@ void adl_serializer::to_json(json & res, const DerivationOutpu overloaded{ [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, [&](const DerivationOutput::CAFixed & dof) { - /* it would be nice to output the path for user convenience, but - this would require us to know the store dir. */ + res = dof.ca; + // FIXME print refs? + /* it would be nice to output the path for user convenience, but + this would require us to know the store dir. */ #if 0 res["path"] = dof.path(store, drvName, outputName); #endif - res["method"] = std::string{dof.ca.method.render()}; - res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); - res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); - // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { res["method"] = std::string{dof.method.render()}; @@ -1341,15 +1339,12 @@ adl_serializer::from_json(const json & _json, const Experiment }; } - else if (keys == (std::set{"method", "hashAlgo", "hash"})) { - auto [method, hashAlgo] = methodAlgo(); + else if (keys == (std::set{"method", "hash"})) { auto dof = DerivationOutput::CAFixed{ - .ca = - ContentAddress{ - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), - }, + .ca = static_cast(_json), }; + if (dof.ca.method == ContentAddressMethod::Raw::Text) + xpSettings.require(Xp::DynamicDerivations, "text-hashed derivation output in JSON"); /* We no longer produce this (denormalized) field (for the reasons described above), so we don't need to check it. */ #if 0 @@ -1392,7 +1387,7 @@ void adl_serializer::to_json(json & res, const Derivation & d) res["name"] = d.name; - res["version"] = 3; + res["version"] = 4; { nlohmann::json & outputsObj = res["outputs"]; @@ -1403,13 +1398,16 @@ void adl_serializer::to_json(json & res, const Derivation & d) } { - auto & inputsList = res["inputSrcs"]; - inputsList = nlohmann::json::array(); - for (auto & input : d.inputSrcs) - inputsList.emplace_back(input); - } + auto & inputsObj = res["inputs"]; + inputsObj = nlohmann::json::object(); + + { + auto & inputsList = inputsObj["srcs"]; + inputsList = nlohmann::json::array(); + for (auto & input : d.inputSrcs) + inputsList.emplace_back(input); + } - { auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { auto value = nlohmann::json::object(); value["outputs"] = inputNode.value; @@ -1421,12 +1419,11 @@ void adl_serializer::to_json(json & res, const Derivation & d) } return value; }; - { - auto & inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { - inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } + + auto & inputDrvsObj = inputsObj["drvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); } } @@ -1449,8 +1446,8 @@ Derivation adl_serializer::from_json(const json & _json, const Exper res.name = getString(valueAt(json, "name")); - if (valueAt(json, "version") != 3) - throw Error("Only derivation format version 3 is currently supported."); + if (valueAt(json, "version") != 4) + throw Error("Only derivation format version 4 is currently supported."); try { auto outputs = getObject(valueAt(json, "outputs")); @@ -1463,32 +1460,39 @@ Derivation adl_serializer::from_json(const json & _json, const Exper } try { - auto inputSrcs = getArray(valueAt(json, "inputSrcs")); - for (auto & input : inputSrcs) - res.inputSrcs.insert(input); - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputSrcs'"); - throw; - } + auto inputsObj = getObject(valueAt(json, "inputs")); - try { - auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { - auto & json = getObject(_json); - DerivedPathMap::ChildNode node; - node.value = getStringSet(valueAt(json, "outputs")); - auto drvs = getObject(valueAt(json, "dynamicOutputs")); - for (auto & [outputId, childNode] : drvs) { - xpSettings.require( - Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); - node.childMap[outputId] = doInput(childNode); - } - return node; - }; - auto drvs = getObject(valueAt(json, "inputDrvs")); - for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); + try { + auto inputSrcs = getArray(valueAt(inputsObj, "srcs")); + for (auto & input : inputSrcs) + res.inputSrcs.insert(input); + } catch (Error & e) { + e.addTrace({}, "while reading key 'srcs'"); + throw; + } + + try { + auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { + auto & json = getObject(_json); + DerivedPathMap::ChildNode node; + node.value = getStringSet(valueAt(json, "outputs")); + auto drvs = getObject(valueAt(json, "dynamicOutputs")); + for (auto & [outputId, childNode] : drvs) { + xpSettings.require( + Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + auto drvs = getObject(valueAt(inputsObj, "drvs")); + for (auto & [inputDrvPath, inputOutputs] : drvs) + res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'drvs'"); + throw; + } } catch (Error & e) { - e.addTrace({}, "while reading key 'inputDrvs'"); + e.addTrace({}, "while reading key 'inputs'"); throw; } diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix index 3c24ac2ee..87f2d9cfe 100644 --- a/tests/functional/dyn-drv/non-trivial.nix +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -51,10 +51,12 @@ builtins.outputOf "$word": "hello, from $word!", "PATH": ${builtins.toJSON path} }, - "inputDrvs": { - $inputDrvs + "inputs": { + "drvs": { + $inputDrvs + }, + "srcs": [] }, - "inputSrcs": [], "name": "build-$word", "outputs": { "out": { @@ -63,7 +65,7 @@ builtins.outputOf } }, "system": "${system}", - "version": 3 + "version": 4 } EOF drvPath=$(echo "$json" | nix derivation add) From caa196e31d00df3fa31d8fc47ae2efb8eb5ac6d4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 17:44:07 -0400 Subject: [PATCH 10/11] Make the store path info `ca` field structured in JSON The old string format is a holdover from the pre JSON days. It is not friendly to users who need to get the information out of it. Also introduce the sort of versioning we have for derivation for this format too. --- doc/manual/source/protocols/json/meson.build | 2 +- ...re-object-info-v1 => store-object-info-v2} | 0 ...info-v1.yaml => store-object-info-v2.yaml} | 31 ++++++++++++++++--- .../protocols/json/store-object-info.md | 12 +++---- src/json-schema-checks/meson.build | 12 +++---- src/libstore-tests/data/nar-info/impure.json | 12 +++++-- src/libstore-tests/data/nar-info/pure.json | 12 +++++-- .../data/path-info/empty_impure.json | 3 +- .../data/path-info/empty_pure.json | 3 +- src/libstore-tests/data/path-info/impure.json | 12 +++++-- src/libstore-tests/data/path-info/pure.json | 12 +++++-- src/libstore/path-info.cc | 24 ++++++++++++-- tests/functional/fixed.sh | 11 ++++++- tests/functional/git-hashing/simple-common.sh | 14 +++++++-- tests/functional/impure-derivations.sh | 2 +- tests/functional/nix-profile.sh | 2 +- tests/functional/signing.sh | 2 +- 17 files changed, 130 insertions(+), 36 deletions(-) rename doc/manual/source/protocols/json/schema/{store-object-info-v1 => store-object-info-v2} (100%) rename doc/manual/source/protocols/json/schema/{store-object-info-v1.yaml => store-object-info-v2.yaml} (91%) diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c0b8416d7..4ab94c63b 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -12,7 +12,7 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v1', + 'store-object-info-v2', 'derivation-v4', 'deriving-path-v1', 'build-trace-entry-v1', diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1 b/doc/manual/source/protocols/json/schema/store-object-info-v2 similarity index 100% rename from doc/manual/source/protocols/json/schema/store-object-info-v1 rename to doc/manual/source/protocols/json/schema/store-object-info-v2 diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml similarity index 91% rename from doc/manual/source/protocols/json/schema/store-object-info-v1.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v2.yaml index d79f25043..4f442e0c3 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml @@ -1,6 +1,6 @@ -"$schema": "http://json-schema.org/draft-07/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v1.json" -title: Store Object Info +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" +title: Store Object Info v2 description: | Information about a [store object](@docroot@/store/store-object.md). @@ -41,11 +41,27 @@ $defs: This is the minimal set of fields that describe what a store object contains. type: object required: + - version - narHash - narSize - references - ca properties: + version: + type: integer + const: 2 + title: Format version (must be 2) + description: | + Must be `2`. + This is a guard that allows us to continue evolving this format. + Here is the rough version history: + + - Version 0: `.narinfo` line-oriented format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from `.narinfo` format. + + - Version 2: Use structured JSON type for `ca` + path: type: string title: Store Path @@ -76,7 +92,10 @@ $defs: type: string ca: - type: ["string", "null"] + oneOf: + - type: "null" + const: null + - "$ref": "./content-address-v1.yaml" title: Content Address description: | If the store object is [content-addressed](@docroot@/store/store-object/content-address.md), @@ -91,6 +110,7 @@ $defs: In other words, the same store object in different stores could have different values for these impure fields. type: object required: + - version - narHash - narSize - references @@ -101,6 +121,7 @@ $defs: - ultimate - signatures properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } @@ -164,6 +185,7 @@ $defs: This download information, being specific to how the store object happens to be stored and transferred, is also considered to be non-intrinsic / impure. type: object required: + - version - narHash - narSize - references @@ -179,6 +201,7 @@ $defs: - downloadHash - downloadSize properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4673dd773..6a101ab0f 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,29 +1,29 @@ -{{#include store-object-info-v1-fixed.md}} +{{#include store-object-info-v2-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v1/pure.json}} +{{#include schema/store-object-info-v2/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v1/impure.json}} +{{#include schema/store-object-info-v2/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v1/empty_pure.json}} +{{#include schema/store-object-info-v2/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v1/empty_impure.json}} +{{#include schema/store-object-info-v2/empty_impure.json}} ``` ### NAR info (minimal) @@ -41,5 +41,5 @@ diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index fedacedeb..f72affb0b 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -134,7 +134,7 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -144,7 +144,7 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -162,7 +162,7 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', 'empty_pure.json', @@ -170,7 +170,7 @@ schemas += [ }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', 'files' : [ 'impure.json', 'empty_impure.json', @@ -178,14 +178,14 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', 'files' : [ 'impure.json', ], diff --git a/src/libstore-tests/data/nar-info/impure.json b/src/libstore-tests/data/nar-info/impure.json index bb9791a6a..f35ff990b 100644 --- a/src/libstore-tests/data/nar-info/impure.json +++ b/src/libstore-tests/data/nar-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "compression": "xz", "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", @@ -16,5 +23,6 @@ "qwer" ], "ultimate": true, - "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 2 } diff --git a/src/libstore-tests/data/nar-info/pure.json b/src/libstore-tests/data/nar-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/nar-info/pure.json +++ b/src/libstore-tests/data/nar-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_impure.json b/src/libstore-tests/data/path-info/empty_impure.json index be982dcef..381acaa03 100644 --- a/src/libstore-tests/data/path-info/empty_impure.json +++ b/src/libstore-tests/data/path-info/empty_impure.json @@ -6,5 +6,6 @@ "references": [], "registrationTime": null, "signatures": [], - "ultimate": false + "ultimate": false, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_pure.json b/src/libstore-tests/data/path-info/empty_pure.json index 10d9f508a..6d3fa646b 100644 --- a/src/libstore-tests/data/path-info/empty_pure.json +++ b/src/libstore-tests/data/path-info/empty_pure.json @@ -2,5 +2,6 @@ "ca": null, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 0, - "references": [] + "references": [], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/impure.json b/src/libstore-tests/data/path-info/impure.json index 0c452cc49..141b38a16 100644 --- a/src/libstore-tests/data/path-info/impure.json +++ b/src/libstore-tests/data/path-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, @@ -12,5 +19,6 @@ "asdf", "qwer" ], - "ultimate": true + "ultimate": true, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/pure.json b/src/libstore-tests/data/path-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/path-info/pure.json +++ b/src/libstore-tests/data/path-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 09a78a4ad..c535d08f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -156,6 +156,8 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf auto jsonObject = json::object(); + jsonObject["version"] = 2; + jsonObject["narHash"] = narHash.to_string(hashFormat, true); jsonObject["narSize"] = narSize; @@ -165,7 +167,7 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf jsonRefs.emplace_back(store.printStorePath(ref)); } - jsonObject["ca"] = ca ? (std::optional{renderContentAddress(*ca)}) : std::nullopt; + jsonObject["ca"] = ca; if (includeImpureInfo) { jsonObject["deriver"] = deriver ? (std::optional{store.printStorePath(*deriver)}) : std::nullopt; @@ -189,6 +191,16 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store }; auto & json = getObject(_json); + + // Check version (optional for backward compatibility) + nlohmann::json::number_unsigned_t version = 1; + if (json.contains("version")) { + version = getUnsigned(valueAt(json, "version")); + if (version != 1 && version != 2) { + throw Error("Unsupported path info JSON format version %d, expected 1 through 2", version); + } + } + res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); res.narSize = getUnsigned(valueAt(json, "narSize")); @@ -205,7 +217,15 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store // missing is for back-compat. if (auto * rawCa0 = optionalValueAt(json, "ca")) if (auto * rawCa = getNullable(*rawCa0)) - res.ca = ContentAddress::parse(getString(*rawCa)); + switch (version) { + case 1: + // old string format also used in SQLite DB and .narinfo + res.ca = ContentAddress::parse(getString(*rawCa)); + break; + case 2 ... std::numeric_limits::max(): + res.ca = *rawCa; + break; + } if (auto * rawDeriver0 = optionalValueAt(json, "deriver")) if (auto * rawDeriver = getNullable(*rawDeriver0)) diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index edf6f88d4..7861392ec 100755 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -14,7 +14,16 @@ nix-build fixed.nix -A bad --no-out-link && fail "should fail" # Building with the bad hash should produce the "good" output path as # a side-effect. [[ -e $path ]] -nix path-info --json "$path" | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd +nix path-info --json "$path" | jq -e \ + --arg hash "$(nix hash convert --to base64 "md5:8ddd8be4b179a529afa5f2ffae4b9858")" \ + '.[].ca == { + method: "flat", + hash: { + algorithm: "md5", + format: "base64", + hash: $hash + }, + }' echo 'testing good...' nix-build fixed.nix -A good --no-out-link diff --git a/tests/functional/git-hashing/simple-common.sh b/tests/functional/git-hashing/simple-common.sh index 08b5c0e71..eaa0a9529 100644 --- a/tests/functional/git-hashing/simple-common.sh +++ b/tests/functional/git-hashing/simple-common.sh @@ -47,9 +47,17 @@ try2 () { hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath") [[ "$hashFromGit" == "$expected" ]] - local caFromNix - caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca") - [[ "fixed:git:$hashAlgo:$(nix hash convert --to nix32 "$hashAlgo:$hashFromGit")" = "$caFromNix" ]] + nix path-info --json "$path" | jq -e \ + --arg algo "$hashAlgo" \ + --arg hash "$(nix hash convert --to base64 "$hashAlgo:$hashFromGit")" \ + '.[].ca == { + method: "git", + hash: { + algorithm: $algo, + format: "base64", + hash: $hash + }, + }' } test0 () { diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index e0b7c3eea..211abccb0 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -30,7 +30,7 @@ path1_stuff=$(echo "$json" | jq -r .[].outputs.stuff) [[ $(< "$path1"/n) = 0 ]] [[ $(< "$path1_stuff"/bla) = 0 ]] -[[ $(nix path-info --json "$path1" | jq .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$path1" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out) [[ $(< "$path2"/n) = 1 ]] diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 922162d4b..494b24ddb 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -166,7 +166,7 @@ printf 4.0 > "$flake1Dir"/version printf Utrecht > "$flake1Dir"/who nix profile add "$flake1Dir" [[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[[ $(nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -r .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' # Override the outputs. nix profile remove simple flake1 diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 2893efec7..1bcaf2f53 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -58,7 +58,7 @@ nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' +nix path-info --json "$outPathCA" | jq -e '.[].ca | .method == "flat" and .hash.algorithm == "md5"' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. From 8cc3ede0fac902c66a9563a9924c54eda66d48c9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 31 Oct 2025 00:43:45 -0400 Subject: [PATCH 11/11] Add change-log entry for derivation format changes --- doc/manual/rl-next/json-format-changes.md | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 doc/manual/rl-next/json-format-changes.md diff --git a/doc/manual/rl-next/json-format-changes.md b/doc/manual/rl-next/json-format-changes.md new file mode 100644 index 000000000..c5518ee1b --- /dev/null +++ b/doc/manual/rl-next/json-format-changes.md @@ -0,0 +1,47 @@ +--- +synopsis: "JSON format changes for store path info and derivations" +prs: [] +issues: [] +--- + +JSON formats for store path info and derivations have been updated with new versions and structured fields. + +## Store Path Info JSON (Version 2) + +The store path info JSON format has been updated from version 1 to version 2: + +- **Added `version` field**: + + All store path info JSON now includes `"version": 2`. + +- **Structured `ca` field**: + + Content address is now a structured JSON object instead of a string: + + - Old: `"ca": "fixed:r:sha256:1abc..."` + - New: `"ca": {"method": "nar", "hash": {"algorithm": "sha256", "format": "base64", "hash": "EMIJ+giQ..."}}` + - Still `null` values for input-addressed store objects + +Version 1 format is still accepted when reading for backward compatibility. + +**Affected command**: `nix path-info --json` + +## Derivation JSON (Version 4) + +The derivation JSON format has been updated from version 3 to version 4: + +- **Restructured inputs**: + + Inputs are now nested under an `inputs` object: + + - Old: `"inputSrcs": [...], "inputDrvs": {...}` + - New: `"inputs": {"srcs": [...], "drvs": {...}}` + +- **Consistent content addresses**: + + Floating content-addressed outputs now use structured JSON format. + This is the same format as `ca` in in store path info (after the new version). + +Version 3 and earlier formats are *not* accepted when reading. + +**Affected command**: `nix derivation`, namely it's `show` and `add` sub-commands.