From 3775a2a2268bbc18716363e38868e3bf76fd3884 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:22:54 +0000 Subject: [PATCH 01/36] build(deps): bump actions/upload-artifact from 4 to 5 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 5. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..18ae4d8bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,13 +125,13 @@ jobs: cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY if: ${{ matrix.instrumented }} - name: Upload coverage reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: coverage-reports path: coverage-reports/ if: ${{ matrix.instrumented }} - name: Upload installer tarball - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: installer-${{matrix.os}} path: out/* From ccc06451df3ca9345977ca4cdf7d412f6603dd90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:35:42 +0000 Subject: [PATCH 02/36] build(deps): bump actions/download-artifact from 5 to 6 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 5 to 6. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..10103847a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Download installer tarball - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: installer-${{matrix.os}} path: out From d8cec03fcec9c7532db18334cf71a6935d94884d Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Sat, 1 Nov 2025 21:38:42 +0000 Subject: [PATCH 03/36] nix flake check: log success in verbose mode The rule of silence can be a little surprising. As a compromise to changing the default behavior, this adds printing a success message in verbose mode, where we don't really have a reason to be silent about our success. --- src/nix/flake.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b826e943c..a6595dc91 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -811,6 +811,8 @@ struct CmdFlakeCheck : FlakeCommand if (hasErrors) throw Error("some errors were encountered during the evaluation"); + logger->log(lvlInfo, ANSI_GREEN "all checks passed!" ANSI_NORMAL); + if (!omittedSystems.empty()) { // TODO: empty system is not visible; render all as nix strings? warn( From 0d7b16da4dae9dc8f07756a6ef87caf17406e835 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Oct 2025 13:03:18 -0400 Subject: [PATCH 04/36] Split realisation protocol unit tests This will allow us to more accurately test dropping support for dependent realisations, by separating the tests that should not change from the tests that should. I do that change in PR #14247, but even if for some reasons we don't end up doing this soon, I think it is still good to separate the test data this way so we have the option of doing that at some point. --- src/libstore-tests/common-protocol.cc | 19 +++++++++++++-- .../common-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/common-protocol/realisation.bin | Bin 520 -> 384 bytes .../serve-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/serve-protocol/realisation.bin | Bin 520 -> 384 bytes .../worker-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/worker-protocol/realisation.bin | Bin 520 -> 384 bytes src/libstore-tests/serve-protocol.cc | 17 +++++++++++++ src/libstore-tests/worker-protocol.cc | 23 +++++++++++++++--- 9 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 src/libstore-tests/data/common-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-with-deps.bin diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 2c001957b..7c40e8cdb 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -114,13 +114,28 @@ CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +CHARACTERIZATION_TEST( + realisation_with_deps, + "realisation-with-deps", + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/data/common-protocol/realisation-with-deps.bin b/src/libstore-tests/data/common-protocol/realisation-with-deps.bin new file mode 100644 index 0000000000000000000000000000000000000000..54a78b64ebcf0726bd7da506434a316c7336e6e9 GIT binary patch literal 320 zcmXqFWB`L|rIgfy)V!3`ypo{Q#GK6H#FEVXykaG*YNg_gL?cr(E3-5UGs`r~)I=i- zBjco$L_;&vR0A^ubF;J*gOpSgV>7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -104,6 +113,14 @@ VERSIONED_CHARACTERIZATION_TEST( .outputName = "baz", }, }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index c4afde3bd..8f70e937b 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -150,13 +150,30 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -172,7 +189,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, }, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, From 144c66215b49bdadba7e11d2fd3ff9b108750274 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 18:01:37 -0400 Subject: [PATCH 05/36] JSON Schema for build trace entry Note, starting to make progress on #11895 by calling it this in the manual. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/build-trace-entry.md | 27 +++++++ doc/manual/source/protocols/json/meson.build | 1 + .../json/schema/build-trace-entry-v1 | 1 + .../json/schema/build-trace-entry-v1.yaml | 74 +++++++++++++++++++ doc/manual/source/store/build-trace.md | 2 +- src/json-schema-checks/build-trace-entry | 1 + src/json-schema-checks/meson.build | 9 +++ src/json-schema-checks/package.nix | 1 + 10 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/protocols/json/build-trace-entry.md create mode 120000 doc/manual/source/protocols/json/schema/build-trace-entry-v1 create mode 100644 doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml create mode 120000 src/json-schema-checks/build-trace-entry diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 7d29df3c3..e13c6f33d 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -37,6 +37,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path + ../../src/libstore-tests/data/realisation ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index b87bf93a3..580076ece 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -126,6 +126,7 @@ - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) + - [Build Trace Entry](protocols/json/build-trace-entry.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md new file mode 100644 index 000000000..8050a2840 --- /dev/null +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -0,0 +1,27 @@ +{{#include build-trace-entry-v1-fixed.md}} + +## Examples + +### Simple build trace entry + +```json +{{#include schema/build-trace-entry-v1/simple.json}} +``` + +### Build trace entry with dependencies + +```json +{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}} +``` + +### Build trace entry with signature + +```json +{{#include schema/build-trace-entry-v1/with-signature.json}} +``` + + \ No newline at end of file diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 7ebcff697..d8e94d68c 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -15,6 +15,7 @@ schemas = [ 'store-object-info-v1', 'derivation-v3', 'deriving-path-v1', + 'build-trace-entry-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1 b/doc/manual/source/protocols/json/schema/build-trace-entry-v1 new file mode 120000 index 000000000..0d02880a5 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/realisation \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml new file mode 100644 index 000000000..cabf2c350 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml @@ -0,0 +1,74 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json" +title: Build Trace Entry +description: | + A record of a successful build outcome for a specific derivation output. + + This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md) entry. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +type: object +required: + - id + - outPath + - dependentRealisations + - signatures +properties: + id: + type: string + title: Derivation Output ID + pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" + description: | + Unique identifier for the derivation output that was built. + + Format: `{hash-quotient-drv}!{output-name}` + + - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). + Begins with `sha256:`. + + - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") + + Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` + + outPath: + "$ref": "store-path-v1.yaml" + title: Output Store Path + description: | + The path to the store object that resulted from building this derivation for the given output name. + + dependentRealisations: + type: object + title: Underlying Base Build Trace + description: | + This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence. + + Keys are derivation output IDs (same format as the main `id` field). + Values are the store paths that those dependencies resolved to. + + As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries. + This is the set of base build trace entries that this derived build trace is derived from. + (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.) + + patternProperties: + "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": + $ref: "store-path-v1.yaml" + title: Dependent Store Path + description: Store path that this dependency resolved to during the build + additionalProperties: false + + signatures: + type: array + title: Build Signatures + description: | + A set of cryptographic signatures attesting to the authenticity of this build trace entry. + items: + type: string + title: Signature + description: A single cryptographic signature + +additionalProperties: false diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index 1086dcb88..8860bc6c7 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -29,7 +29,7 @@ And even in that case, a different result doesn't mean the original entry was a As such, the decision of whether to trust a counterparty's build trace is a fundamentally subject policy choice. Build trace entries are typically *signed* in order to enable arbitrary public-key-based trust polices. -## Derived build traces +## Derived build traces {#derived} Implementations that wish to memoize the above may also keep additional *derived* build trace entries that do map unresolved derivations. But if they do so, they *must* also keep the underlying base entries with resolved derivation keys around. diff --git a/src/json-schema-checks/build-trace-entry b/src/json-schema-checks/build-trace-entry new file mode 120000 index 000000000..9175e750e --- /dev/null +++ b/src/json-schema-checks/build-trace-entry @@ -0,0 +1 @@ +../../src/libstore-tests/data/realisation \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 67f553162..c2c7fbff4 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -54,6 +54,15 @@ schemas = [ 'single_built_built.json', ], }, + { + 'stem' : 'build-trace-entry', + 'schema' : schema_dir / 'build-trace-entry-v1.yaml', + 'files' : [ + 'simple.json', + 'with-dependent-realisations.json', + 'with-signature.json', + ], + }, ] # Derivation and Derivation output diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 160db003f..057a6e85b 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -23,6 +23,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path + ../../src/libstore-tests/data/realisation ../../src/libstore-tests/data/derivation ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info From c3d4c5f69d93278dafe2c631f955ffe0e47ca689 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 22:00:54 +0000 Subject: [PATCH 06/36] build(deps): bump cachix/install-nix-action from 31.5.1 to 31.8.2 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.5.1 to 31.8.2. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/c134e4c9e34bac6cab09cf239815f9339aaaf84e...456688f15bc354bef6d396e4a35f4f89d40bf2b7) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.8.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..60c617978 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 + - uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31.8.2 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} From 389bcba97a1295440a24c887840b1af3e73f0dd3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 13:42:52 -0400 Subject: [PATCH 07/36] JSON Impl and schema for `BuildResult` --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/json/build-result.md | 21 +++ doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/build-result-v1 | 1 + .../json/schema/build-result-v1.yaml | 136 +++++++++++++++++ src/json-schema-checks/build-result | 1 + src/json-schema-checks/meson.build | 9 ++ src/json-schema-checks/package.nix | 1 + src/libstore-tests/build-result.cc | 108 +++++++++++++ .../data/build-result/not-deterministic.json | 9 ++ .../data/build-result/output-rejected.json | 9 ++ .../data/build-result/success.json | 23 +++ src/libstore-tests/meson.build | 1 + src/libstore/build-result.cc | 142 ++++++++++++++++++ .../include/nix/store/build-result.hh | 3 + 16 files changed, 467 insertions(+) create mode 100644 doc/manual/source/protocols/json/build-result.md create mode 120000 doc/manual/source/protocols/json/schema/build-result-v1 create mode 100644 doc/manual/source/protocols/json/schema/build-result-v1.yaml create mode 120000 src/json-schema-checks/build-result create mode 100644 src/libstore-tests/build-result.cc create mode 100644 src/libstore-tests/data/build-result/not-deterministic.json create mode 100644 src/libstore-tests/data/build-result/output-rejected.json create mode 100644 src/libstore-tests/data/build-result/success.json diff --git a/doc/manual/package.nix b/doc/manual/package.nix index e13c6f33d..343e40016 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -41,6 +41,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 580076ece..5be3d6a90 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -127,6 +127,7 @@ - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) + - [Build Result](protocols/json/build-result.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/build-result.md b/doc/manual/source/protocols/json/build-result.md new file mode 100644 index 000000000..527e7bcc0 --- /dev/null +++ b/doc/manual/source/protocols/json/build-result.md @@ -0,0 +1,21 @@ +{{#include build-result-v1-fixed.md}} + +## Examples + +### Successful build + +```json +{{#include schema/build-result-v1/success.json}} +``` + +### Failed build (output rejected) + +```json +{{#include schema/build-result-v1/output-rejected.json}} +``` + +### Failed build (non-deterministic) + +```json +{{#include schema/build-result-v1/not-deterministic.json}} +``` \ No newline at end of file diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index d8e94d68c..c56de49c7 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -16,6 +16,7 @@ schemas = [ 'derivation-v3', 'deriving-path-v1', 'build-trace-entry-v1', + 'build-result-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-result-v1 b/doc/manual/source/protocols/json/schema/build-result-v1 new file mode 120000 index 000000000..a143d2c50 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml new file mode 100644 index 000000000..31f59a44d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -0,0 +1,136 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-result-v1.json" +title: Build Result +description: | + This schema describes the JSON representation of Nix's `BuildResult` type, which represents the result of building a derivation or substituting store paths. + + Build results can represent either successful builds (with built outputs) or various types of failures. + +oneOf: + - "$ref": "#/$defs/success" + - "$ref": "#/$defs/failure" +type: object +required: + - success + - status +properties: + timesBuilt: + type: integer + minimum: 0 + title: Times built + description: | + How many times this build was performed. + + startTime: + type: integer + minimum: 0 + title: Start time + description: | + The start time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + stopTime: + type: integer + minimum: 0 + title: Stop time + description: | + The stop time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + cpuUser: + type: integer + minimum: 0 + title: User CPU time + description: | + User CPU time the build took, in microseconds. + + cpuSystem: + type: integer + minimum: 0 + title: System CPU time + description: | + System CPU time the build took, in microseconds. + +"$defs": + success: + type: object + title: Successful Build Result + description: | + Represents a successful build with built outputs. + required: + - success + - status + - builtOutputs + properties: + success: + const: true + title: Success indicator + description: | + Always true for successful build results. + + status: + type: string + title: Success status + description: | + Status string for successful builds. + enum: + - "Built" + - "Substituted" + - "AlreadyValid" + - "ResolvesToAlreadyValid" + + builtOutputs: + type: object + title: Built outputs + description: | + A mapping from output names to their build trace entries. + additionalProperties: + "$ref": "build-trace-entry-v1.yaml" + + failure: + type: object + title: Failed Build Result + description: | + Represents a failed build with error information. + required: + - success + - status + - errorMsg + properties: + success: + const: false + title: Success indicator + description: | + Always false for failed build results. + + status: + type: string + title: Failure status + description: | + Status string for failed builds. + enum: + - "PermanentFailure" + - "InputRejected" + - "OutputRejected" + - "TransientFailure" + - "CachedFailure" + - "TimedOut" + - "MiscFailure" + - "DependencyFailed" + - "LogLimitExceeded" + - "NotDeterministic" + - "NoSubstituters" + - "HashMismatch" + + errorMsg: + type: string + title: Error message + description: | + Information about the error if the build failed. + + isNonDeterministic: + type: boolean + title: Non-deterministic flag + description: | + If timesBuilt > 1, whether some builds did not produce the same result. + + Note that 'isNonDeterministic = false' does not mean the build is deterministic, + just that we don't have evidence of non-determinism. diff --git a/src/json-schema-checks/build-result b/src/json-schema-checks/build-result new file mode 120000 index 000000000..8010d0fdd --- /dev/null +++ b/src/json-schema-checks/build-result @@ -0,0 +1 @@ +../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index c2c7fbff4..65a2651b7 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -150,6 +150,15 @@ schemas += [ 'impure.json', ], }, + { + 'stem' : 'build-result', + 'schema' : schema_dir / 'build-result-v1.yaml', + 'files' : [ + 'success.json', + 'output-rejected.json', + 'not-deterministic.json', + ], + }, # Match exact variant { 'stem' : 'store-object-info', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 057a6e85b..5365fe75e 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -28,6 +28,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result ./. ]; diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc new file mode 100644 index 000000000..85e799c2a --- /dev/null +++ b/src/libstore-tests/build-result.cc @@ -0,0 +1,108 @@ +#include + +#include "nix/store/build-result.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class BuildResultTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "build-result"; + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct BuildResultJsonTest : BuildResultTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(BuildResultJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(BuildResultJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +using namespace std::literals::chrono_literals; + +INSTANTIATE_TEST_SUITE_P( + BuildResultJSON, + BuildResultJsonTest, + ::testing::Values( + std::pair{ + "not-deterministic", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = false, // Note: This field is separate from the status + }}, + .timesBuilt = 1, + }, + }, + std::pair{ + "output-rejected", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + .isNonDeterministic = false, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + }, + std::pair{ + "success", + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs{ + { + "foo", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + }, + }, + { + "bar", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + }, + }, + }, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + })); + +} // namespace nix diff --git a/src/libstore-tests/data/build-result/not-deterministic.json b/src/libstore-tests/data/build-result/not-deterministic.json new file mode 100644 index 000000000..c24a15795 --- /dev/null +++ b/src/libstore-tests/data/build-result/not-deterministic.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "NotDeterministic", + "stopTime": 0, + "success": false, + "timesBuilt": 1 +} diff --git a/src/libstore-tests/data/build-result/output-rejected.json b/src/libstore-tests/data/build-result/output-rejected.json new file mode 100644 index 000000000..9494bf4ec --- /dev/null +++ b/src/libstore-tests/data/build-result/output-rejected.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 30, + "status": "OutputRejected", + "stopTime": 50, + "success": false, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json new file mode 100644 index 000000000..4baadb547 --- /dev/null +++ b/src/libstore-tests/data/build-result/success.json @@ -0,0 +1,23 @@ +{ + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 4d464ad89..f76df8bcb 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -54,6 +54,7 @@ deps_private += gtest subdir('nix-meson-build-support/common') sources = files( + 'build-result.cc', 'common-protocol.cc', 'content-address.cc', 'derivation-advanced-attrs.cc', diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index ecbd27b49..e3d9e9085 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -1,4 +1,6 @@ #include "nix/store/build-result.hh" +#include "nix/util/json-utils.hh" +#include namespace nix { @@ -11,4 +13,144 @@ std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Succes bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default; std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default; +static constexpr std::array, 4> successStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Success::e, #e} + ENUM_ENTRY(Built), + ENUM_ENTRY(Substituted), + ENUM_ENTRY(AlreadyValid), + ENUM_ENTRY(ResolvesToAlreadyValid), +#undef ENUM_ENTRY +}}; + +static std::string_view successStatusToString(BuildResult::Success::Status status) +{ + for (const auto & [enumVal, str] : successStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown success status: %d", static_cast(status)); +} + +static BuildResult::Success::Status successStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : successStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result success status '%s'", str); +} + +static constexpr std::array, 12> failureStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Failure::e, #e} + ENUM_ENTRY(PermanentFailure), + ENUM_ENTRY(InputRejected), + ENUM_ENTRY(OutputRejected), + ENUM_ENTRY(TransientFailure), + ENUM_ENTRY(CachedFailure), + ENUM_ENTRY(TimedOut), + ENUM_ENTRY(MiscFailure), + ENUM_ENTRY(DependencyFailed), + ENUM_ENTRY(LogLimitExceeded), + ENUM_ENTRY(NotDeterministic), + ENUM_ENTRY(NoSubstituters), + ENUM_ENTRY(HashMismatch), +#undef ENUM_ENTRY +}}; + +static std::string_view failureStatusToString(BuildResult::Failure::Status status) +{ + for (const auto & [enumVal, str] : failureStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown failure status: %d", static_cast(status)); +} + +static BuildResult::Failure::Status failureStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : failureStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result failure status '%s'", str); +} + } // namespace nix + +namespace nlohmann { + +using namespace nix; + +void adl_serializer::to_json(json & res, const BuildResult & br) +{ + res = json::object(); + + // Common fields + res["timesBuilt"] = br.timesBuilt; + res["startTime"] = br.startTime; + res["stopTime"] = br.stopTime; + + if (br.cpuUser.has_value()) { + res["cpuUser"] = br.cpuUser->count(); + } + if (br.cpuSystem.has_value()) { + res["cpuSystem"] = br.cpuSystem->count(); + } + + // Handle success or failure variant + std::visit( + overloaded{ + [&](const BuildResult::Success & success) { + res["success"] = true; + res["status"] = successStatusToString(success.status); + res["builtOutputs"] = success.builtOutputs; + }, + [&](const BuildResult::Failure & failure) { + res["success"] = false; + res["status"] = failureStatusToString(failure.status); + res["errorMsg"] = failure.errorMsg; + res["isNonDeterministic"] = failure.isNonDeterministic; + }, + }, + br.inner); +} + +BuildResult adl_serializer::from_json(const json & _json) +{ + auto & json = getObject(_json); + + BuildResult br; + + // Common fields + br.timesBuilt = getUnsigned(valueAt(json, "timesBuilt")); + br.startTime = getUnsigned(valueAt(json, "startTime")); + br.stopTime = getUnsigned(valueAt(json, "stopTime")); + + if (auto cpuUser = optionalValueAt(json, "cpuUser")) { + br.cpuUser = std::chrono::microseconds(getUnsigned(*cpuUser)); + } + if (auto cpuSystem = optionalValueAt(json, "cpuSystem")) { + br.cpuSystem = std::chrono::microseconds(getUnsigned(*cpuSystem)); + } + + // Determine success or failure based on success field + bool success = getBoolean(valueAt(json, "success")); + std::string statusStr = getString(valueAt(json, "status")); + + if (success) { + BuildResult::Success s; + s.status = successStatusFromString(statusStr); + s.builtOutputs = valueAt(json, "builtOutputs"); + br.inner = std::move(s); + } else { + BuildResult::Failure f; + f.status = failureStatusFromString(statusStr); + f.errorMsg = getString(valueAt(json, "errorMsg")); + f.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")); + br.inner = std::move(f); + } + + return br; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index 0446c4038..4739232f8 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -7,6 +7,7 @@ #include "nix/store/derived-path.hh" #include "nix/store/realisation.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -175,3 +176,5 @@ struct KeyedBuildResult : BuildResult }; } // namespace nix + +JSON_IMPL(nix::BuildResult) From 469123eda19028ea784b78b6d21ae0d4e2c91ab3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 14:10:13 +0200 Subject: [PATCH 08/36] doc: Check link fragments with lychee --- flake.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a70617b74..897889a71 100644 --- a/flake.nix +++ b/flake.nix @@ -320,7 +320,16 @@ checks = forAllSystems ( system: - (import ./ci/gha/tests { + let + pkgs = nixpkgsFor.${system}.native; + in + { + # https://nixos.org/manual/nixpkgs/stable/index.html#tester-lycheeLinkCheck + linkcheck = pkgs.testers.lycheeLinkCheck { + site = self.packages.${system}.nix-manual + "/share/doc/nix/manual"; + }; + } + // (import ./ci/gha/tests { inherit system; pkgs = nixpkgsFor.${system}.native; nixFlake = self; From ae15d4eaf395f02a6b08e75044d5b5d48e1cb12c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 4 Nov 2025 00:18:51 +0100 Subject: [PATCH 09/36] Fix links in the manual --- doc/manual/anchors.jq | 2 +- doc/manual/source/command-ref/nix-channel.md | 2 +- .../source/command-ref/nix-env/upgrade.md | 2 +- doc/manual/source/development/building.md | 4 ++-- doc/manual/source/development/testing.md | 2 +- doc/manual/source/glossary.md | 10 +++++----- .../source/language/advanced-attributes.md | 2 +- doc/manual/source/language/derivations.md | 2 +- doc/manual/source/language/identifiers.md | 2 +- doc/manual/source/language/index.md | 10 +++++----- doc/manual/source/language/string-context.md | 2 +- .../source/language/string-interpolation.md | 2 +- doc/manual/source/language/syntax.md | 6 +++--- .../source/protocols/json/schema/hash-v1.yaml | 2 +- .../source/protocols/nix-archive/index.md | 2 +- doc/manual/source/release-notes/rl-2.18.md | 2 +- doc/manual/source/release-notes/rl-2.19.md | 4 ++-- doc/manual/source/release-notes/rl-2.23.md | 2 +- doc/manual/source/release-notes/rl-2.24.md | 2 +- doc/manual/source/store/building.md | 2 +- doc/manual/source/store/derivation/index.md | 2 +- .../source/store/derivation/outputs/index.md | 2 +- .../store/store-object/content-address.md | 4 ++-- doc/manual/source/store/store-path.md | 2 +- src/libexpr/primops.cc | 20 +++++++++---------- src/libexpr/primops/context.cc | 4 ++-- src/libstore/include/nix/store/globals.hh | 2 +- src/libstore/include/nix/store/local-store.hh | 2 +- src/libutil/experimental-features.cc | 2 +- src/nix/flake.md | 2 +- 30 files changed, 53 insertions(+), 53 deletions(-) diff --git a/doc/manual/anchors.jq b/doc/manual/anchors.jq index 72309779c..4ee2bc130 100755 --- a/doc/manual/anchors.jq +++ b/doc/manual/anchors.jq @@ -3,7 +3,7 @@ def transform_anchors_html: - . | gsub($empty_anchor_regex; "") + . | gsub($empty_anchor_regex; "") | gsub($anchor_regex; "" + .text + ""); diff --git a/doc/manual/source/command-ref/nix-channel.md b/doc/manual/source/command-ref/nix-channel.md index ed9cbb41f..3d02a7d40 100644 --- a/doc/manual/source/command-ref/nix-channel.md +++ b/doc/manual/source/command-ref/nix-channel.md @@ -14,7 +14,7 @@ The moving parts of channels are: - The official channels listed at - The user-specific list of [subscribed channels](#subscribed-channels) - The [downloaded channel contents](#channels) -- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-I) or the [`NIX_PATH` environment variable](#env-NIX_PATH) > **Note** > diff --git a/doc/manual/source/command-ref/nix-env/upgrade.md b/doc/manual/source/command-ref/nix-env/upgrade.md index 2779363c3..bf4c1a8ed 100644 --- a/doc/manual/source/command-ref/nix-env/upgrade.md +++ b/doc/manual/source/command-ref/nix-env/upgrade.md @@ -22,7 +22,7 @@ left untouched; this is not an error. It is also not an error if an element of *args* matches no installed derivations. For a description of how *args* is mapped to a set of store paths, see -[`--install`](#operation---install). If *args* describes multiple +[`--install`](./install.md). If *args* describes multiple store paths with the same symbolic name, only the one with the highest version is installed. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 889d81d80..eb65a7247 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -66,7 +66,7 @@ You can also build Nix for one of the [supported platforms](#platforms). This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. [`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command +[`nix-command`]: @docroot@/development/experimental-features.md#xp-feature-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -256,7 +256,7 @@ You can use any of the other supported environments in place of `nix-cli-ccacheS ## Editor integration The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. -See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). +See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#building-nix-with-flakes) or in [classic Nix](#building-nix). To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. Meson's configure always produces this inside the build directory. diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index c0b130155..7c2cbbb5d 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -119,7 +119,7 @@ This will: 3. Stop the program when the test fails, allowing the user to then issue arbitrary commands to GDB. -### Characterisation testing { #characaterisation-testing-unit } +### Characterisation testing { #characterisation-testing-unit } See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index e6a294e7d..502e6d4de 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -208,7 +208,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -279,7 +279,7 @@ See [References](@docroot@/store/store-object.md#references) for details. -- [referrer]{#gloss-reference} +- [referrer]{#gloss-referrer} A reversed edge from one [store object] to another. @@ -367,8 +367,8 @@ Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. - The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#attribute-set) containing: - - attributes that refer to the files of a package, typically in the form of [derivation outputs](#output), + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), - attributes with metadata, such as information about how the package is supposed to be used. The exact shape of these attribute sets is up to convention. @@ -383,7 +383,7 @@ [string]: ./language/types.md#type-string [path]: ./language/types.md#type-path - [attribute name]: ./language/types.md#attribute-set + [attribute name]: ./language/types.md#type-attrs - [base directory]{#gloss-base-directory} diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index c9d64f060..f0b1a4c73 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -333,7 +333,7 @@ Here is more information on the `output*` attributes, and what values they may b `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format, because in that case the choice of hash algorithm is determined by `outputHash`. - - [`outputHash`]{#adv-attr-outputHashAlgo}; [`outputHash`]{#adv-attr-outputHashMode} + - [`outputHash`]{#adv-attr-outputHash} This will specify the output hash of the single output of a [fixed-output derivation]. diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 43eec680b..2403183fc 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -16,7 +16,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. - See [derivation outputs](@docroot@/store/derivation/index.md#outputs) for what this is affects. + See [derivation outputs](@docroot@/store/derivation/outputs/index.md#outputs) for what this is affects. [store path]: @docroot@/store/store-path.md diff --git a/doc/manual/source/language/identifiers.md b/doc/manual/source/language/identifiers.md index 584a2f861..67bb1eeec 100644 --- a/doc/manual/source/language/identifiers.md +++ b/doc/manual/source/language/identifiers.md @@ -16,7 +16,7 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A *name* can be written as an [identifier](#identifier) or a [string literal](./string-literals.md). +A *name* can be written as an [identifier](#identifiers) or a [string literal](./string-literals.md). > **Syntax** > diff --git a/doc/manual/source/language/index.md b/doc/manual/source/language/index.md index 1eb14e96d..116f928dc 100644 --- a/doc/manual/source/language/index.md +++ b/doc/manual/source/language/index.md @@ -137,7 +137,7 @@ This is an incomplete overview of language features, by example. - [Booleans](@docroot@/language/types.md#type-boolean) + [Booleans](@docroot@/language/types.md#type-bool) @@ -245,7 +245,7 @@ This is an incomplete overview of language features, by example. - An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#type-attrs) with attributes named `x` and `y` @@ -285,7 +285,7 @@ This is an incomplete overview of language features, by example. - [Lists](@docroot@/language/types.md#list) with three elements. + [Lists](@docroot@/language/types.md#type-list) with three elements. @@ -369,7 +369,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#type-attrs) (evaluates to `1`) @@ -381,7 +381,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#type-attrs) with default (evaluates to `3`) diff --git a/doc/manual/source/language/string-context.md b/doc/manual/source/language/string-context.md index 0d8fcdefa..65c59d865 100644 --- a/doc/manual/source/language/string-context.md +++ b/doc/manual/source/language/string-context.md @@ -111,7 +111,7 @@ It creates an [attribute set] representing the string context, which can be insp [`builtins.hasContext`]: ./builtins.md#builtins-hasContext [`builtins.getContext`]: ./builtins.md#builtins-getContext -[attribute set]: ./types.md#attribute-set +[attribute set]: ./types.md#type-attrs ## Clearing string contexts diff --git a/doc/manual/source/language/string-interpolation.md b/doc/manual/source/language/string-interpolation.md index a503d5f04..8e25d2b63 100644 --- a/doc/manual/source/language/string-interpolation.md +++ b/doc/manual/source/language/string-interpolation.md @@ -6,7 +6,7 @@ Such a construct is called *interpolated string*, and the expression inside is a [string]: ./types.md#type-string [path]: ./types.md#type-path -[attribute set]: ./types.md#attribute-set +[attribute set]: ./types.md#type-attrs > **Syntax** > diff --git a/doc/manual/source/language/syntax.md b/doc/manual/source/language/syntax.md index 85162db74..b127aca14 100644 --- a/doc/manual/source/language/syntax.md +++ b/doc/manual/source/language/syntax.md @@ -51,7 +51,7 @@ See [String literals](string-literals.md). Path literals can also include [string interpolation], besides being [interpolated into other expressions]. - [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expression At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. @@ -235,7 +235,7 @@ of object-oriented programming, for example. ## Recursive sets -Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. +Recursive sets are like normal [attribute sets](./types.md#type-attrs), but the attributes can refer to each other. > *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` @@ -287,7 +287,7 @@ This evaluates to `"foobar"`. ## Inheriting attributes -When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +When defining an [attribute set](./types.md#type-attrs) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). This can be shortened using the `inherit` keyword. Example: diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml index 316fb6d73..821546dee 100644 --- a/doc/manual/source/protocols/json/schema/hash-v1.yaml +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -51,4 +51,4 @@ additionalProperties: false description: | The hash algorithm used to compute the hash value. - `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake-hashing) experimental feature. + `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake3-hashes) experimental feature. diff --git a/doc/manual/source/protocols/nix-archive/index.md b/doc/manual/source/protocols/nix-archive/index.md index 4d25f63e2..bd2a8e833 100644 --- a/doc/manual/source/protocols/nix-archive/index.md +++ b/doc/manual/source/protocols/nix-archive/index.md @@ -4,7 +4,7 @@ This is the complete specification of the [Nix Archive] format. The Nix Archive format closely follows the abstract specification of a [file system object] tree, because it is designed to serialize exactly that data structure. -[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive [file system object]: @docroot@/store/file-system-object.md The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. diff --git a/doc/manual/source/release-notes/rl-2.18.md b/doc/manual/source/release-notes/rl-2.18.md index eb26fc9e7..71b25f408 100644 --- a/doc/manual/source/release-notes/rl-2.18.md +++ b/doc/manual/source/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/language/advanced-attributes.md#adv-attr-unsafeDiscardReferences) attribute is no longer guarded by an experimental flag and can be used freely. diff --git a/doc/manual/source/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md index 06c704324..04f8c9c28 100644 --- a/doc/manual/source/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-feature-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes). - The interface for creating and updating lock files has been overhauled: diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index e6b0e9ffc..b358a0fdc 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. diff --git a/doc/manual/source/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md index d4af3cb51..e9b46bb22 100644 --- a/doc/manual/source/release-notes/rl-2.24.md +++ b/doc/manual/source/release-notes/rl-2.24.md @@ -93,7 +93,7 @@ - Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) - Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/conf-file.md#conf-min-free) to set the minimum free space to 1 gigabyte. This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index dbfe6b5ca..f2d470e99 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -8,7 +8,7 @@ - Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. -## Builder Execution +## Builder Execution {#builder-execution} The [`builder`](./derivation/index.md#builder) is executed as follows: diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 61c5335ff..670f3b2bd 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -102,7 +102,7 @@ But rather than somehow scanning all the other fields for inputs, Nix requires t ### System {#system} -The system type on which the [`builder`](#attr-builder) executable is meant to be run. +The system type on which the [`builder`](#builder) executable is meant to be run. A necessary condition for Nix to schedule a given derivation on some [Nix instance] is for the "system" of that derivation to match that instance's [`system` configuration option] or [`extra-platforms` configuration option]. diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index 0683f5703..ca2ce6665 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -43,7 +43,7 @@ In particular, the specification decides: - if the content is content-addressed, how is it content addressed -- if the content is content-addressed, [what is its content address](./content-address.md#fixed-content-addressing) (and thus what is its [store path]) +- if the content is content-addressed, [what is its content address](./content-address.md#fixed) (and thus what is its [store path]) ## Types of derivations diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 36e841fa3..7834ac510 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -1,7 +1,7 @@ # Content-Addressing Store Objects Just [like][fso-ca] [File System Objects][File System Object], -[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-address), unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. @@ -107,7 +107,7 @@ References (to other store objects and self-references alike) are supported so l > > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. -This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. +This uses the corresponding [Git](../file-system-object/content-address.md#git) method of file system object content addressing. References are not supported. diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index beec2389b..4061f3653 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -6,7 +6,7 @@ > > A rendered store path -Nix implements references to [store objects](./index.md#store-object) as *store paths*. +Nix implements references to [store objects](./store-object.md) as *store paths*. Think of a store path as an [opaque], [unique identifier]: The only way to obtain store path is by adding or building store objects. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 96e79fedd..d1aae64fa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -825,10 +825,10 @@ static RegisterPrimOp primop_genericClosure( - [Int](@docroot@/language/types.md#type-int) - [Float](@docroot@/language/types.md#type-float) - - [Boolean](@docroot@/language/types.md#type-boolean) + - [Boolean](@docroot@/language/types.md#type-bool) - [String](@docroot@/language/types.md#type-string) - [Path](@docroot@/language/types.md#type-path) - - [List](@docroot@/language/types.md#list) + - [List](@docroot@/language/types.md#type-list) The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added. Items are compared by their `key` attribute. @@ -2103,7 +2103,7 @@ static RegisterPrimOp primop_findFile( builtins.findFile builtins.nixPath "nixpkgs" ``` - A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + A search path is represented as a list of [attribute sets](./types.md#type-attrs) with two attributes: - `prefix` is a relative path. - `path` denotes a file system location @@ -2395,7 +2395,7 @@ static RegisterPrimOp primop_outputOf({ returns an input placeholder for the output of the output of `myDrv`. - This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-paths), e.g. as part of installable syntax on the command line. + This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-path), e.g. as part of installable syntax on the command line. )", .fun = prim_outputOf, .experimentalFeature = Xp::DynamicDerivations, @@ -4966,7 +4966,7 @@ static RegisterPrimOp primop_compareVersions({ version *s1* is older than version *s2*, `0` if they are the same, and `1` if *s1* is newer than *s2*. The version comparison algorithm is the same as the one used by [`nix-env - -u`](../command-ref/nix-env.md#operation---upgrade). + -u`](../command-ref/nix-env/upgrade.md). )", .fun = prim_compareVersions, }); @@ -4995,7 +4995,7 @@ static RegisterPrimOp primop_splitVersion({ .doc = R"( Split a string representing a version into its components, by the same version splitting logic underlying the version comparison in - [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade). + [`nix-env -u`](../command-ref/nix-env/upgrade.md). )", .fun = prim_splitVersion, }); @@ -5045,9 +5045,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) Primitive value. It can be returned by - [comparison operators](@docroot@/language/operators.md#Comparison) + [comparison operators](@docroot@/language/operators.md#comparison) and used in - [conditional expressions](@docroot@/language/syntax.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#conditionals). The name `true` is not special, and can be shadowed: @@ -5068,9 +5068,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) Primitive value. It can be returned by - [comparison operators](@docroot@/language/operators.md#Comparison) + [comparison operators](@docroot@/language/operators.md#comparison) and used in - [conditional expressions](@docroot@/language/syntax.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#conditionals). The name `false` is not special, and can be shadowed: diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 12b8ffdf9..8a9fe42e8 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -79,7 +79,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency( Create a copy of the given string where every [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) string context element is turned into a - [constant](@docroot@/language/string-context.md#string-context-element-constant) + [constant](@docroot@/language/string-context.md#string-context-constant) string context element. This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). @@ -145,7 +145,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies( .args = {"s"}, .doc = R"( Create a copy of the given string where a single - [constant](@docroot@/language/string-context.md#string-context-element-constant) + [constant](@docroot@/language/string-context.md#string-context-constant) string context element is turned into a [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) string context element. diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 8aa82c4a2..5ddfbee30 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -189,7 +189,7 @@ public: 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism. diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c56de49c7..c0b8416d7 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -13,7 +13,7 @@ schemas = [ 'content-address-v1', 'store-path-v1', 'store-object-info-v1', - 'derivation-v3', + 'derivation-v4', 'deriving-path-v1', 'build-trace-entry-v1', 'build-result-v1', diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml similarity index 78% rename from doc/manual/source/protocols/json/schema/derivation-v3.yaml rename to doc/manual/source/protocols/json/schema/derivation-v4.yaml index fa68adcb1..2528f7502 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -1,8 +1,8 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v4.json" title: Derivation description: | - Experimental JSON representation of a Nix derivation (version 3). + Experimental JSON representation of a Nix derivation (version 4). This schema describes the JSON representation of Nix's `Derivation` type. @@ -17,8 +17,7 @@ required: - name - version - outputs - - inputSrcs - - inputDrvs + - inputs - system - builder - args @@ -32,10 +31,10 @@ properties: Used when calculating store paths for the derivation’s outputs. version: - const: 3 - title: Format version (must be 3) + const: 4 + title: Format version (must be 4) description: | - Must be `3`. + Must be `4`. This is a guard that allows us to continue evolving this format. The choice of `3` is fairly arbitrary, but corresponds to this informal version: @@ -47,6 +46,12 @@ properties: - Version 3: Drop store dir from store paths, just include base name. + - Version 4: Two cleanups, batched together to lesson churn: + + - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) + + - Use canonical content address JSON format for floating content addressed derivation outputs. + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. outputs: @@ -70,47 +75,56 @@ properties: additionalProperties: "$ref": "#/$defs/output/overall" - inputSrcs: - type: array - title: Input source paths - description: | - List of store paths on which this derivation depends. - - > **Example** - > - > ```json - > "inputSrcs": [ - > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", - > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" - > ] - > ``` - items: - $ref: "store-path-v1.yaml" - - inputDrvs: + inputs: type: object - title: Input derivations + title: Derivation inputs description: | - Mapping of derivation paths to lists of output names they provide. - - > **Example** - > - > ```json - > "inputDrvs": { - > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] - > } - > ``` - > - > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. - patternProperties: - "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": - title: Store Path + Input dependencies for the derivation, organized into source paths and derivation dependencies. + required: + - srcs + - drvs + properties: + srcs: + type: array + title: Input source paths description: | - A store path to a derivation, mapped to the outputs of that derivation. - oneOf: - - "$ref": "#/$defs/outputNames" - - "$ref": "#/$defs/dynamicOutputs" + List of store paths on which this derivation depends. + + > **Example** + > + > ```json + > "srcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + items: + $ref: "store-path-v1.yaml" + drvs: + type: object + title: Input derivations + description: | + Mapping of derivation paths to lists of output names they provide. + + > **Example** + > + > ```json + > "drvs": { + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > } + > ``` + > + > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + patternProperties: + "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": + title: Store Path + description: | + A store path to a derivation, mapped to the outputs of that derivation. + oneOf: + - "$ref": "#/$defs/outputNames" + - "$ref": "#/$defs/dynamicOutputs" + additionalProperties: false additionalProperties: false system: @@ -189,24 +203,18 @@ properties: The output is content-addressed, and the content-address is fixed in advance. See [Fixed-output content-addressing](@docroot@/store/derivation/outputs/content-address.md#fixed) for more details. - type: object + "$ref": "./content-address-v1.yaml" required: - method - - hashAlgo - hash properties: method: - "$ref": "./content-address-v1.yaml#/$defs/method" description: | Method of content addressing used for this output. - hashAlgo: - title: Hash algorithm - "$ref": "./hash-v1.yaml#/$defs/algorithm" hash: - type: string title: Expected hash value description: | - The expected content hash in base-16. + The expected content hash. additionalProperties: false caFloating: diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 65a2651b7..fedacedeb 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -70,7 +70,7 @@ schemas += [ # Match overall { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml', + 'schema' : schema_dir / 'derivation-v4.yaml', 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', @@ -78,7 +78,7 @@ schemas += [ }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/overall', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/overall', 'files' : [ 'output-caFixedFlat.json', 'output-caFixedNAR.json', @@ -92,14 +92,14 @@ schemas += [ # Match exact variant { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/inputAddressed', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/inputAddressed', 'files' : [ 'output-inputAddressed.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFixed', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFixed', 'files' : [ 'output-caFixedFlat.json', 'output-caFixedNAR.json', @@ -108,21 +108,21 @@ schemas += [ }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/caFloating', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFloating', 'files' : [ 'output-caFloating.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/deferred', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/deferred', 'files' : [ 'output-deferred.json', ], }, { 'stem' : 'derivation', - 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output/impure', + 'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/impure', 'files' : [ 'output-impure.json', ], diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json index eb4bd4f3d..781b4cb14 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -12,8 +12,10 @@ "outputHashMode": "recursive", "system": "my-system" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-defaults", "outputs": { "out": { @@ -22,5 +24,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 3a4a3079b..7437b51ef 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -8,8 +8,10 @@ "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { @@ -33,5 +35,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index b10355af7..2a4e70558 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -9,25 +9,27 @@ "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, - "inputDrvs": { - "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] }, - "inputSrcs": [ - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" - ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { @@ -101,5 +103,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index d66882036..55dbe62e0 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -25,25 +25,27 @@ "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" }, - "inputDrvs": { - "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] }, - "inputSrcs": [ - "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" - ], "name": "advanced-attributes", "outputs": { "out": { @@ -52,5 +54,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ca/self-contained.json b/src/libstore-tests/data/derivation/ca/self-contained.json index 331beb7be..c05710140 100644 --- a/src/libstore-tests/data/derivation/ca/self-contained.json +++ b/src/libstore-tests/data/derivation/ca/self-contained.json @@ -10,8 +10,10 @@ "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", "system": "x86_64-linux" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "myname", "outputs": { "out": { @@ -20,5 +22,5 @@ } }, "system": "x86_64-linux", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/dyn-dep-derivation.json b/src/libstore-tests/data/derivation/dyn-dep-derivation.json index 1a9f54c53..1793c5f2d 100644 --- a/src/libstore-tests/data/derivation/dyn-dep-derivation.json +++ b/src/libstore-tests/data/derivation/dyn-dep-derivation.json @@ -7,33 +7,35 @@ "env": { "BIG_BAD": "WOLF" }, - "inputDrvs": { - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": [ - "kitten" - ] + "inputs": { + "drvs": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } }, - "goose": { - "dynamicOutputs": {}, - "outputs": [ - "gosling" - ] - } - }, - "outputs": [ - "cat", - "dog" - ] - } + "outputs": [ + "cat", + "dog" + ] + } + }, + "srcs": [ + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ] }, - "inputSrcs": [ - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], "name": "dyn-dep-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json index 0fa543f21..898762123 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json @@ -10,8 +10,10 @@ "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", "system": "my-system" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-defaults", "outputs": { "out": { @@ -19,5 +21,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index e02392ea1..c51095986 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -8,8 +8,10 @@ "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { @@ -29,5 +31,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index 9230b06b6..e07d1294b 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -9,25 +9,27 @@ "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev", "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, - "inputDrvs": { - "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] }, - "inputSrcs": [ - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" - ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { @@ -96,5 +98,5 @@ "system": "my-system" }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index ba5911c91..372b4fbb9 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -23,25 +23,27 @@ "requiredSystemFeatures": "rainbow uid-range", "system": "my-system" }, - "inputDrvs": { - "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] + "inputs": { + "drvs": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } }, - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "dev", - "out" - ] - } + "srcs": [ + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] }, - "inputSrcs": [ - "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" - ], "name": "advanced-attributes", "outputs": { "out": { @@ -49,5 +51,5 @@ } }, "system": "my-system", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json index e6a0123f6..9a38608b3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedFlat.json +++ b/src/libstore-tests/data/derivation/output-caFixedFlat.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "flat" } diff --git a/src/libstore-tests/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json index b57e065a9..767c605a3 100644 --- a/src/libstore-tests/data/derivation/output-caFixedNAR.json +++ b/src/libstore-tests/data/derivation/output-caFixedNAR.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "nar" } diff --git a/src/libstore-tests/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json index 84778509e..a04f1ff2a 100644 --- a/src/libstore-tests/data/derivation/output-caFixedText.json +++ b/src/libstore-tests/data/derivation/output-caFixedText.json @@ -1,5 +1,8 @@ { - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "sha256", + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8=" + }, "method": "text" } diff --git a/src/libstore-tests/data/derivation/simple-derivation.json b/src/libstore-tests/data/derivation/simple-derivation.json index 41a049aef..04129a096 100644 --- a/src/libstore-tests/data/derivation/simple-derivation.json +++ b/src/libstore-tests/data/derivation/simple-derivation.json @@ -7,20 +7,22 @@ "env": { "BIG_BAD": "WOLF" }, - "inputDrvs": { - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } + "inputs": { + "drvs": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "srcs": [ + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ] }, - "inputSrcs": [ - "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], "name": "simple-derivation", "outputs": {}, "system": "wasm-sel4", - "version": 3 + "version": 4 } diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 228b8069f..bf411053a 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -636,7 +636,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) auto outj_ph = nix::hashPlaceholder("outj"); std::string drvJson = R"({ - "version": 3, + "version": 4, "name": "multi-output-test", "system": ")" + nix::settings.thisSystem.get() + R"(", @@ -668,8 +668,10 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) "outa": ")" + outa_ph + R"(" }, - "inputDrvs": {}, - "inputSrcs": [], + "inputs": { + "drvs": {}, + "srcs": [] + }, "outputs": { "outd": { "hashAlgo": "sha256", "method": "nar" }, "outf": { "hashAlgo": "sha256", "method": "nar" }, diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index e6ac08fd9..31ca167f9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1293,15 +1293,13 @@ void adl_serializer::to_json(json & res, const DerivationOutpu overloaded{ [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, [&](const DerivationOutput::CAFixed & dof) { - /* it would be nice to output the path for user convenience, but - this would require us to know the store dir. */ + res = dof.ca; + // FIXME print refs? + /* it would be nice to output the path for user convenience, but + this would require us to know the store dir. */ #if 0 res["path"] = dof.path(store, drvName, outputName); #endif - res["method"] = std::string{dof.ca.method.render()}; - res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); - res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); - // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { res["method"] = std::string{dof.method.render()}; @@ -1341,15 +1339,12 @@ adl_serializer::from_json(const json & _json, const Experiment }; } - else if (keys == (std::set{"method", "hashAlgo", "hash"})) { - auto [method, hashAlgo] = methodAlgo(); + else if (keys == (std::set{"method", "hash"})) { auto dof = DerivationOutput::CAFixed{ - .ca = - ContentAddress{ - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), - }, + .ca = static_cast(_json), }; + if (dof.ca.method == ContentAddressMethod::Raw::Text) + xpSettings.require(Xp::DynamicDerivations, "text-hashed derivation output in JSON"); /* We no longer produce this (denormalized) field (for the reasons described above), so we don't need to check it. */ #if 0 @@ -1392,7 +1387,7 @@ void adl_serializer::to_json(json & res, const Derivation & d) res["name"] = d.name; - res["version"] = 3; + res["version"] = 4; { nlohmann::json & outputsObj = res["outputs"]; @@ -1403,13 +1398,16 @@ void adl_serializer::to_json(json & res, const Derivation & d) } { - auto & inputsList = res["inputSrcs"]; - inputsList = nlohmann::json::array(); - for (auto & input : d.inputSrcs) - inputsList.emplace_back(input); - } + auto & inputsObj = res["inputs"]; + inputsObj = nlohmann::json::object(); + + { + auto & inputsList = inputsObj["srcs"]; + inputsList = nlohmann::json::array(); + for (auto & input : d.inputSrcs) + inputsList.emplace_back(input); + } - { auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { auto value = nlohmann::json::object(); value["outputs"] = inputNode.value; @@ -1421,12 +1419,11 @@ void adl_serializer::to_json(json & res, const Derivation & d) } return value; }; - { - auto & inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { - inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } + + auto & inputDrvsObj = inputsObj["drvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); } } @@ -1449,8 +1446,8 @@ Derivation adl_serializer::from_json(const json & _json, const Exper res.name = getString(valueAt(json, "name")); - if (valueAt(json, "version") != 3) - throw Error("Only derivation format version 3 is currently supported."); + if (valueAt(json, "version") != 4) + throw Error("Only derivation format version 4 is currently supported."); try { auto outputs = getObject(valueAt(json, "outputs")); @@ -1463,32 +1460,39 @@ Derivation adl_serializer::from_json(const json & _json, const Exper } try { - auto inputSrcs = getArray(valueAt(json, "inputSrcs")); - for (auto & input : inputSrcs) - res.inputSrcs.insert(input); - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputSrcs'"); - throw; - } + auto inputsObj = getObject(valueAt(json, "inputs")); - try { - auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { - auto & json = getObject(_json); - DerivedPathMap::ChildNode node; - node.value = getStringSet(valueAt(json, "outputs")); - auto drvs = getObject(valueAt(json, "dynamicOutputs")); - for (auto & [outputId, childNode] : drvs) { - xpSettings.require( - Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); - node.childMap[outputId] = doInput(childNode); - } - return node; - }; - auto drvs = getObject(valueAt(json, "inputDrvs")); - for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); + try { + auto inputSrcs = getArray(valueAt(inputsObj, "srcs")); + for (auto & input : inputSrcs) + res.inputSrcs.insert(input); + } catch (Error & e) { + e.addTrace({}, "while reading key 'srcs'"); + throw; + } + + try { + auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { + auto & json = getObject(_json); + DerivedPathMap::ChildNode node; + node.value = getStringSet(valueAt(json, "outputs")); + auto drvs = getObject(valueAt(json, "dynamicOutputs")); + for (auto & [outputId, childNode] : drvs) { + xpSettings.require( + Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + auto drvs = getObject(valueAt(inputsObj, "drvs")); + for (auto & [inputDrvPath, inputOutputs] : drvs) + res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'drvs'"); + throw; + } } catch (Error & e) { - e.addTrace({}, "while reading key 'inputDrvs'"); + e.addTrace({}, "while reading key 'inputs'"); throw; } diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix index 3c24ac2ee..87f2d9cfe 100644 --- a/tests/functional/dyn-drv/non-trivial.nix +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -51,10 +51,12 @@ builtins.outputOf "$word": "hello, from $word!", "PATH": ${builtins.toJSON path} }, - "inputDrvs": { - $inputDrvs + "inputs": { + "drvs": { + $inputDrvs + }, + "srcs": [] }, - "inputSrcs": [], "name": "build-$word", "outputs": { "out": { @@ -63,7 +65,7 @@ builtins.outputOf } }, "system": "${system}", - "version": 3 + "version": 4 } EOF drvPath=$(echo "$json" | nix derivation add) From caa196e31d00df3fa31d8fc47ae2efb8eb5ac6d4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 17:44:07 -0400 Subject: [PATCH 34/36] Make the store path info `ca` field structured in JSON The old string format is a holdover from the pre JSON days. It is not friendly to users who need to get the information out of it. Also introduce the sort of versioning we have for derivation for this format too. --- doc/manual/source/protocols/json/meson.build | 2 +- ...re-object-info-v1 => store-object-info-v2} | 0 ...info-v1.yaml => store-object-info-v2.yaml} | 31 ++++++++++++++++--- .../protocols/json/store-object-info.md | 12 +++---- src/json-schema-checks/meson.build | 12 +++---- src/libstore-tests/data/nar-info/impure.json | 12 +++++-- src/libstore-tests/data/nar-info/pure.json | 12 +++++-- .../data/path-info/empty_impure.json | 3 +- .../data/path-info/empty_pure.json | 3 +- src/libstore-tests/data/path-info/impure.json | 12 +++++-- src/libstore-tests/data/path-info/pure.json | 12 +++++-- src/libstore/path-info.cc | 24 ++++++++++++-- tests/functional/fixed.sh | 11 ++++++- tests/functional/git-hashing/simple-common.sh | 14 +++++++-- tests/functional/impure-derivations.sh | 2 +- tests/functional/nix-profile.sh | 2 +- tests/functional/signing.sh | 2 +- 17 files changed, 130 insertions(+), 36 deletions(-) rename doc/manual/source/protocols/json/schema/{store-object-info-v1 => store-object-info-v2} (100%) rename doc/manual/source/protocols/json/schema/{store-object-info-v1.yaml => store-object-info-v2.yaml} (91%) diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c0b8416d7..4ab94c63b 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -12,7 +12,7 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v1', + 'store-object-info-v2', 'derivation-v4', 'deriving-path-v1', 'build-trace-entry-v1', diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1 b/doc/manual/source/protocols/json/schema/store-object-info-v2 similarity index 100% rename from doc/manual/source/protocols/json/schema/store-object-info-v1 rename to doc/manual/source/protocols/json/schema/store-object-info-v2 diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml similarity index 91% rename from doc/manual/source/protocols/json/schema/store-object-info-v1.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v2.yaml index d79f25043..4f442e0c3 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml @@ -1,6 +1,6 @@ -"$schema": "http://json-schema.org/draft-07/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v1.json" -title: Store Object Info +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" +title: Store Object Info v2 description: | Information about a [store object](@docroot@/store/store-object.md). @@ -41,11 +41,27 @@ $defs: This is the minimal set of fields that describe what a store object contains. type: object required: + - version - narHash - narSize - references - ca properties: + version: + type: integer + const: 2 + title: Format version (must be 2) + description: | + Must be `2`. + This is a guard that allows us to continue evolving this format. + Here is the rough version history: + + - Version 0: `.narinfo` line-oriented format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from `.narinfo` format. + + - Version 2: Use structured JSON type for `ca` + path: type: string title: Store Path @@ -76,7 +92,10 @@ $defs: type: string ca: - type: ["string", "null"] + oneOf: + - type: "null" + const: null + - "$ref": "./content-address-v1.yaml" title: Content Address description: | If the store object is [content-addressed](@docroot@/store/store-object/content-address.md), @@ -91,6 +110,7 @@ $defs: In other words, the same store object in different stores could have different values for these impure fields. type: object required: + - version - narHash - narSize - references @@ -101,6 +121,7 @@ $defs: - ultimate - signatures properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } @@ -164,6 +185,7 @@ $defs: This download information, being specific to how the store object happens to be stored and transferred, is also considered to be non-intrinsic / impure. type: object required: + - version - narHash - narSize - references @@ -179,6 +201,7 @@ $defs: - downloadHash - downloadSize properties: + version: { $ref: "#/$defs/base/properties/version" } path: { $ref: "#/$defs/base/properties/path" } narHash: { $ref: "#/$defs/base/properties/narHash" } narSize: { $ref: "#/$defs/base/properties/narSize" } diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4673dd773..6a101ab0f 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,29 +1,29 @@ -{{#include store-object-info-v1-fixed.md}} +{{#include store-object-info-v2-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v1/pure.json}} +{{#include schema/store-object-info-v2/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v1/impure.json}} +{{#include schema/store-object-info-v2/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v1/empty_pure.json}} +{{#include schema/store-object-info-v2/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v1/empty_impure.json}} +{{#include schema/store-object-info-v2/empty_impure.json}} ``` ### NAR info (minimal) @@ -41,5 +41,5 @@ diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index fedacedeb..f72affb0b 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -134,7 +134,7 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -144,7 +144,7 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml', + 'schema' : schema_dir / 'store-object-info-v2.yaml', 'files' : [ 'pure.json', 'impure.json', @@ -162,7 +162,7 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', 'empty_pure.json', @@ -170,7 +170,7 @@ schemas += [ }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', 'files' : [ 'impure.json', 'empty_impure.json', @@ -178,14 +178,14 @@ schemas += [ }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', 'files' : [ 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v1.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', 'files' : [ 'impure.json', ], diff --git a/src/libstore-tests/data/nar-info/impure.json b/src/libstore-tests/data/nar-info/impure.json index bb9791a6a..f35ff990b 100644 --- a/src/libstore-tests/data/nar-info/impure.json +++ b/src/libstore-tests/data/nar-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "compression": "xz", "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", @@ -16,5 +23,6 @@ "qwer" ], "ultimate": true, - "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 2 } diff --git a/src/libstore-tests/data/nar-info/pure.json b/src/libstore-tests/data/nar-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/nar-info/pure.json +++ b/src/libstore-tests/data/nar-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_impure.json b/src/libstore-tests/data/path-info/empty_impure.json index be982dcef..381acaa03 100644 --- a/src/libstore-tests/data/path-info/empty_impure.json +++ b/src/libstore-tests/data/path-info/empty_impure.json @@ -6,5 +6,6 @@ "references": [], "registrationTime": null, "signatures": [], - "ultimate": false + "ultimate": false, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/empty_pure.json b/src/libstore-tests/data/path-info/empty_pure.json index 10d9f508a..6d3fa646b 100644 --- a/src/libstore-tests/data/path-info/empty_pure.json +++ b/src/libstore-tests/data/path-info/empty_pure.json @@ -2,5 +2,6 @@ "ca": null, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 0, - "references": [] + "references": [], + "version": 2 } diff --git a/src/libstore-tests/data/path-info/impure.json b/src/libstore-tests/data/path-info/impure.json index 0c452cc49..141b38a16 100644 --- a/src/libstore-tests/data/path-info/impure.json +++ b/src/libstore-tests/data/path-info/impure.json @@ -1,5 +1,12 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, @@ -12,5 +19,6 @@ "asdf", "qwer" ], - "ultimate": true + "ultimate": true, + "version": 2 } diff --git a/src/libstore-tests/data/path-info/pure.json b/src/libstore-tests/data/path-info/pure.json index 955baec31..2c5cb3bde 100644 --- a/src/libstore-tests/data/path-info/pure.json +++ b/src/libstore-tests/data/path-info/pure.json @@ -1,9 +1,17 @@ { - "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=" + }, + "method": "nar" + }, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ] + ], + "version": 2 } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 09a78a4ad..c535d08f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -156,6 +156,8 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf auto jsonObject = json::object(); + jsonObject["version"] = 2; + jsonObject["narHash"] = narHash.to_string(hashFormat, true); jsonObject["narSize"] = narSize; @@ -165,7 +167,7 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig & store, bool includeImpureInf jsonRefs.emplace_back(store.printStorePath(ref)); } - jsonObject["ca"] = ca ? (std::optional{renderContentAddress(*ca)}) : std::nullopt; + jsonObject["ca"] = ca; if (includeImpureInfo) { jsonObject["deriver"] = deriver ? (std::optional{store.printStorePath(*deriver)}) : std::nullopt; @@ -189,6 +191,16 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store }; auto & json = getObject(_json); + + // Check version (optional for backward compatibility) + nlohmann::json::number_unsigned_t version = 1; + if (json.contains("version")) { + version = getUnsigned(valueAt(json, "version")); + if (version != 1 && version != 2) { + throw Error("Unsupported path info JSON format version %d, expected 1 through 2", version); + } + } + res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); res.narSize = getUnsigned(valueAt(json, "narSize")); @@ -205,7 +217,15 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig & store // missing is for back-compat. if (auto * rawCa0 = optionalValueAt(json, "ca")) if (auto * rawCa = getNullable(*rawCa0)) - res.ca = ContentAddress::parse(getString(*rawCa)); + switch (version) { + case 1: + // old string format also used in SQLite DB and .narinfo + res.ca = ContentAddress::parse(getString(*rawCa)); + break; + case 2 ... std::numeric_limits::max(): + res.ca = *rawCa; + break; + } if (auto * rawDeriver0 = optionalValueAt(json, "deriver")) if (auto * rawDeriver = getNullable(*rawDeriver0)) diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index edf6f88d4..7861392ec 100755 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -14,7 +14,16 @@ nix-build fixed.nix -A bad --no-out-link && fail "should fail" # Building with the bad hash should produce the "good" output path as # a side-effect. [[ -e $path ]] -nix path-info --json "$path" | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd +nix path-info --json "$path" | jq -e \ + --arg hash "$(nix hash convert --to base64 "md5:8ddd8be4b179a529afa5f2ffae4b9858")" \ + '.[].ca == { + method: "flat", + hash: { + algorithm: "md5", + format: "base64", + hash: $hash + }, + }' echo 'testing good...' nix-build fixed.nix -A good --no-out-link diff --git a/tests/functional/git-hashing/simple-common.sh b/tests/functional/git-hashing/simple-common.sh index 08b5c0e71..eaa0a9529 100644 --- a/tests/functional/git-hashing/simple-common.sh +++ b/tests/functional/git-hashing/simple-common.sh @@ -47,9 +47,17 @@ try2 () { hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath") [[ "$hashFromGit" == "$expected" ]] - local caFromNix - caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca") - [[ "fixed:git:$hashAlgo:$(nix hash convert --to nix32 "$hashAlgo:$hashFromGit")" = "$caFromNix" ]] + nix path-info --json "$path" | jq -e \ + --arg algo "$hashAlgo" \ + --arg hash "$(nix hash convert --to base64 "$hashAlgo:$hashFromGit")" \ + '.[].ca == { + method: "git", + hash: { + algorithm: $algo, + format: "base64", + hash: $hash + }, + }' } test0 () { diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index e0b7c3eea..211abccb0 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -30,7 +30,7 @@ path1_stuff=$(echo "$json" | jq -r .[].outputs.stuff) [[ $(< "$path1"/n) = 0 ]] [[ $(< "$path1_stuff"/bla) = 0 ]] -[[ $(nix path-info --json "$path1" | jq .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$path1" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out) [[ $(< "$path2"/n) = 1 ]] diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 922162d4b..494b24ddb 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -166,7 +166,7 @@ printf 4.0 > "$flake1Dir"/version printf Utrecht > "$flake1Dir"/who nix profile add "$flake1Dir" [[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[[ $(nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -r .[].ca) =~ fixed:r:sha256: ]] +nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -e '.[].ca | .method == "nar" and .hash.algorithm == "sha256"' # Override the outputs. nix profile remove simple flake1 diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 2893efec7..1bcaf2f53 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -58,7 +58,7 @@ nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' +nix path-info --json "$outPathCA" | jq -e '.[].ca | .method == "flat" and .hash.algorithm == "md5"' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. From 8cc3ede0fac902c66a9563a9924c54eda66d48c9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 31 Oct 2025 00:43:45 -0400 Subject: [PATCH 35/36] Add change-log entry for derivation format changes --- doc/manual/rl-next/json-format-changes.md | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 doc/manual/rl-next/json-format-changes.md diff --git a/doc/manual/rl-next/json-format-changes.md b/doc/manual/rl-next/json-format-changes.md new file mode 100644 index 000000000..c5518ee1b --- /dev/null +++ b/doc/manual/rl-next/json-format-changes.md @@ -0,0 +1,47 @@ +--- +synopsis: "JSON format changes for store path info and derivations" +prs: [] +issues: [] +--- + +JSON formats for store path info and derivations have been updated with new versions and structured fields. + +## Store Path Info JSON (Version 2) + +The store path info JSON format has been updated from version 1 to version 2: + +- **Added `version` field**: + + All store path info JSON now includes `"version": 2`. + +- **Structured `ca` field**: + + Content address is now a structured JSON object instead of a string: + + - Old: `"ca": "fixed:r:sha256:1abc..."` + - New: `"ca": {"method": "nar", "hash": {"algorithm": "sha256", "format": "base64", "hash": "EMIJ+giQ..."}}` + - Still `null` values for input-addressed store objects + +Version 1 format is still accepted when reading for backward compatibility. + +**Affected command**: `nix path-info --json` + +## Derivation JSON (Version 4) + +The derivation JSON format has been updated from version 3 to version 4: + +- **Restructured inputs**: + + Inputs are now nested under an `inputs` object: + + - Old: `"inputSrcs": [...], "inputDrvs": {...}` + - New: `"inputs": {"srcs": [...], "drvs": {...}}` + +- **Consistent content addresses**: + + Floating content-addressed outputs now use structured JSON format. + This is the same format as `ca` in in store path info (after the new version). + +Version 3 and earlier formats are *not* accepted when reading. + +**Affected command**: `nix derivation`, namely it's `show` and `add` sub-commands. From e6afa20c6754b424bec2ff283df4667e609bbb16 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Nov 2025 18:53:27 -0500 Subject: [PATCH 36/36] Encapsulate and slightly optimize string contexts These steps are done (originally in order, but I squashed it as the end result is still pretty small, and the churn in the code comments was a bit annoying to keep straight). 1. Create proper struct type for string contexts on the heap This will make it easier to change this type in the future. 2. Make `Value::StringWithContext` iterable This make some for loops a lot more terse. 3. Encapsulate `Value::StringWithContext::Context::elems` It turns out the iterators we just exposed are sufficient. 4. Make `StringWithContext::Context` length-prefixed instead Rather than having a null pointer at the end, have a `size_t` at the beginning. This is the exact same size (note that null pointer is longer than null byte) and thus takes no more space! Also, see the new TODO on naming. The thing we already so-named is a builder type for string contexts, not the on-heap type. The `fromBuilder` static method reflects what the names ought to be too. --- src/libexpr/eval-cache.cc | 10 ++-- src/libexpr/eval.cc | 29 +++++----- src/libexpr/include/nix/expr/value.hh | 54 +++++++++++++++++-- src/libexpr/include/nix/expr/value/context.hh | 13 +++++ 4 files changed, 87 insertions(+), 19 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index de74d2143..0372d6cc1 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -136,17 +136,19 @@ struct AttrDb }); } - AttrId setString(AttrKey key, std::string_view s, const char ** context = nullptr) + AttrId setString(AttrKey key, std::string_view s, const Value::StringWithContext::Context * context = nullptr) { return doSQLite([&]() { auto state(_state->lock()); if (context) { std::string ctx; - for (const char ** p = context; *p; ++p) { - if (p != context) + bool first = true; + for (auto * elem : *context) { + if (!first) ctx.push_back(' '); - ctx.append(*p); + ctx.append(elem); + first = false; } state->insertAttributeWithContext.use()(key.first)(symbols[key.second])(AttrType::String) (s) (ctx) .exec(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7036df957..47880b9c5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -821,15 +821,18 @@ void Value::mkString(std::string_view s) mkStringNoCopy(makeImmutableString(s)); } -static const char ** encodeContext(const NixStringContext & context) +Value::StringWithContext::Context * Value::StringWithContext::Context::fromBuilder(const NixStringContext & context) { if (!context.empty()) { - size_t n = 0; - auto ctx = (const char **) allocBytes((context.size() + 1) * sizeof(char *)); - for (auto & i : context) { - ctx[n++] = makeImmutableString({i.to_string()}); + auto ctx = (Value::StringWithContext::Context *) allocBytes(sizeof(size_t) + context.size() * sizeof(char *)); + ctx->size = context.size(); + /* Mapping the original iterator to turn references into + pointers is necessary to make sure that enumerate doesn't + accidently copy the elements when it returns tuples by value. + */ + for (auto [n, i] : enumerate(context | std::views::transform([](const auto & r) { return &r; }))) { + ctx->elems[n] = makeImmutableString({i->to_string()}); } - ctx[n] = nullptr; return ctx; } else return nullptr; @@ -837,12 +840,12 @@ static const char ** encodeContext(const NixStringContext & context) void Value::mkString(std::string_view s, const NixStringContext & context) { - mkStringNoCopy(makeImmutableString(s), encodeContext(context)); + mkStringNoCopy(makeImmutableString(s), Value::StringWithContext::Context::fromBuilder(context)); } void Value::mkStringMove(const char * s, const NixStringContext & context) { - mkStringNoCopy(s, encodeContext(context)); + mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context)); } void Value::mkPath(const SourcePath & path) @@ -2287,9 +2290,9 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string void copyContext(const Value & v, NixStringContext & context, const ExperimentalFeatureSettings & xpSettings) { - if (v.context()) - for (const char ** p = v.context(); *p; ++p) - context.insert(NixStringContextElem::parse(*p, xpSettings)); + if (auto * ctx = v.context()) + for (auto * elem : *ctx) + context.insert(NixStringContextElem::parse(elem, xpSettings)); } std::string_view EvalState::forceString( @@ -2309,7 +2312,9 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s auto s = forceString(v, pos, errorCtx); if (v.context()) { error( - "the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]) + "the string '%1%' is not allowed to refer to a store path (such as '%2%')", + v.string_view(), + *v.context()->begin()) .withTrace(pos, errorCtx) .debugThrow(); } diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 706a4fe3f..c3a45d4d2 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -220,7 +220,55 @@ struct ValueBase struct StringWithContext { const char * c_str; - const char ** context; // must be in sorted order + + /** + * The type of the context itself. + * + * Currently, it is length-prefixed array of pointers to + * null-terminated strings. The strings are specially formatted + * to represent a flattening of the recursive sum type that is a + * context element. + * + * @See NixStringContext for an more easily understood type, + * that of the "builder" for this data structure. + */ + struct Context + { + using Elem = const char *; + + /** + * Number of items in the array + */ + size_t size; + + private: + /** + * must be in sorted order + */ + Elem elems[/*size*/]; + public: + + const Elem * begin() const + { + return elems; + } + + const Elem * end() const + { + return &elems[size]; + } + + /** + * @note returns a null pointer to more concisely encode the + * empty context + */ + static Context * fromBuilder(const NixStringContext & context); + }; + + /** + * May be null for a string without context. + */ + const Context * context; }; struct Path @@ -991,7 +1039,7 @@ public: setStorage(b); } - void mkStringNoCopy(const char * s, const char ** context = 0) noexcept + void mkStringNoCopy(const char * s, const Value::StringWithContext::Context * context = nullptr) noexcept { setStorage(StringWithContext{.c_str = s, .context = context}); } @@ -1117,7 +1165,7 @@ public: return getStorage().c_str; } - const char ** context() const noexcept + const Value::StringWithContext::Context * context() const noexcept { return getStorage().context; } diff --git a/src/libexpr/include/nix/expr/value/context.hh b/src/libexpr/include/nix/expr/value/context.hh index dcfacbb21..625adc37b 100644 --- a/src/libexpr/include/nix/expr/value/context.hh +++ b/src/libexpr/include/nix/expr/value/context.hh @@ -24,6 +24,14 @@ public: } }; +/** + * @todo This should be reamed to `StringContextBuilderElem`, since: + * + * 1. We use `*Builder` for off-heap temporary data structures + * + * 2. The `Nix*` is totally redundant. (And my mistake from a long time + * ago.) + */ struct NixStringContextElem { /** @@ -77,6 +85,11 @@ struct NixStringContextElem std::string to_string() const; }; +/** + * @todo This should be renamed to `StringContextBuilder`. + * + * @see NixStringContextElem for explanation why. + */ typedef std::set NixStringContext; } // namespace nix